[NRL-853] Move backup infrastructure TF state into S3 and add README. Fixup account bootstrap script to work on non-mgmt accounts

Author: mattdean3-nhs

scripts/bootstrap.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# Setup mgmt and non-mgmt AWS accounts for NRLF
+set -o errexit -o nounset -o pipefail
 
 AWS_REGION_NAME="eu-west-2"
 PROFILE_PREFIX="nhsd-nrlf"
@@ -32,18 +34,12 @@ function _check_mgmt() {
 }
 
 function _check_non_mgmt() {
-    if [[ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" != 'nhsd-ddc-spine-nrlf-mgmt' ]]; then
+  if [[ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" == 'nhsd-ddc-spine-nrlf-mgmt' ]]; then
     echo "Please log in as a non-mgmt account" >&2
     return 1
   fi
 }
 
-function _get_mgmt_account(){
-  if ! _check_mgmt; then return 1; fi
-  return $(aws sts get-caller-identity --query Account --output text)
-}
-
-
 function _bootstrap() {
   local command=$1
   local admin_policy_arn="arn:aws:iam::aws:policy/AdministratorAccess"
@@ -55,7 +51,7 @@ function _bootstrap() {
     "create-mgmt")
       _check_mgmt || return 1
 
-      cd $root/terraform/bootstrap/mgmt
+      cd terraform/bootstrap/mgmt
       aws s3api create-bucket --bucket "${truststore_bucket_name}" --region us-east-1 --create-bucket-configuration LocationConstraint="${AWS_REGION_NAME}"
       aws s3api create-bucket --bucket "${state_bucket_name}" --region us-east-1 --create-bucket-configuration LocationConstraint="${AWS_REGION_NAME}"
       aws s3api put-public-access-block --bucket "${state_bucket_name}" --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
@@ -69,7 +65,7 @@ function _bootstrap() {
     "delete-mgmt")
       _check_mgmt || return 1
 
-      cd $root/terraform/bootstrap/mgmt
+      cd terraform/bootstrap/mgmt
       aws dynamodb delete-table --table-name "${state_lock_table_name}" || return 1
       local versioned_objects
       versioned_objects=$(aws s3api list-object-versions \
@@ -90,10 +86,20 @@ function _bootstrap() {
     "create-non-mgmt")
       _check_non_mgmt || return 1
 
-      cd $root/terraform/bootstrap/non-mgmt
+      cd terraform/bootstrap/non-mgmt
       local tf_assume_role_policy
       local mgmt_account_id
-      mgmt_account_id=$(_get_mgmt_account)
+
+      set +e
+        mgmt_account_id=$(aws secretsmanager get-secret-value --secret-id "${MGMT_ACCOUNT_ID_LOCATION}" --query SecretString --output text)
+
+        if [ "${mgmt_account_id}" == "" ]; then
+          aws secretsmanager create-secret --name "${MGMT_ACCOUNT_ID_LOCATION}"
+          echo "Please set ${MGMT_ACCOUNT_ID_LOCATION} in the Secrets Manager and rerun the script"
+          exit 1
+        fi
+      set -e
+
       tf_assume_role_policy=$(awk "{sub(/REPLACEME/,\"${mgmt_account_id}\")}1" terraform-trust-policy.json)
       aws iam create-role --role-name "${TERRAFORM_ROLE_NAME}" --assume-role-policy-document "${tf_assume_role_policy}" || return 1
       aws iam attach-role-policy --policy-arn "${admin_policy_arn}" --role-name "${TERRAFORM_ROLE_NAME}" || return 1

terraform/backup-infrastructure/README.md

@@ -0,0 +1,87 @@
+# NRLF Backup Infrastructure
+
+This directory contains AWS backup terraform resources which are global to a given account.
+
+Each subdirectory corresponds to each AWS account (`prod` and `test`).
+
+**Backup infrastructure should be deployed manually and not be run as part of CI.**
+
+## Table of Contents
+
+1. [Prerequisites](#prerequisites)
+2. [Initialise shell environment](#initialise-shell-environment)
+3. [Deploy backup resources](#deploy-backup-resources)
+4. [Tear down backup resources](#tear-down-backup-resources)
+
+## Prerequisites
+
+Before deploying the NRLF backup infrastructure, you will need:
+
+- An AWS backup account that have already been bootstrapped, as described in [bootstrap/README.md](../bootstrap/README.md). This is a one-time account setup step.
+
+## Deploy backup resources
+
+To deploy the backup resources, first login to the AWS mgmt account on the CLI.
+
+Then, initialise the terraform backup workspace. For the test account:
+
+```shell
+$ cd test
+$ terraform init && ( \
+    terraform workspace new backup-infra-test || \
+    terraform workspace select backup-infra-test )
+```
+
+If you want to apply changes to prod, use the `prod` directory and the `backup-infra-prod` terraform workspace.
+
+Once you have your workspace set, you can plan your changes with:
+
+```shell
+$ terraform plan \
+    -var 'source_account_id=SOURCE_ACCOUNT_ID" \
+    -var 'assume_account=AWS_ACCOUNT_ID' \
+    -var 'assume_role=terraform'
+```
+
+Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account.
+
+Once you're happy with your planned changes, you can apply them with:
+
+```shell
+$ terraform apply \
+    -var 'source_account_id=SOURCE_ACCOUNT_ID" \
+    -var 'assume_account=AWS_ACCOUNT_ID' \
+    -var 'assume_role=terraform'
+```
+
+Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account.
+
+## Tear down backup resources
+
+WARNING - This action will destroy all backup resources from the AWS account. This should
+only be done if you are sure that this is safe and are sure that you are signed into the correct
+AWS account.
+
+To tear down backup resources, first login to the AWS mgmt account on the CLI.
+
+Then, initialise your terraform workspace. For the test account:
+
+```shell
+$ cd test
+$ terraform init && ( \
+    terraform workspace new backup-infra-test || \
+    terraform workspace select backup-infra-test )
+```
+
+If you want to destroy resources in prod, use the `prod` directory and the `backup-infra-prod` terraform workspace.
+
+And then, to tear down:
+
+```shell
+$ terraform destroy \
+    -var 'source_account_id=SOURCE_ACCOUNT_ID" \
+    -var 'assume_account=AWS_ACCOUNT_ID' \
+    -var 'assume_role=terraform'
+```
+
+Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account.

terraform/backup-infrastructure/test/data.tf

@@ -1,5 +1 @@
-data "aws_arn" "source_terraform_role" {
-  arn = var.source_terraform_role_arn
-}
-
 data "aws_caller_identity" "current" {}

terraform/backup-infrastructure/test/locals.tf

@@ -1,8 +1,8 @@
 locals {
   # Adjust these as required
   project_name     = "nrlf-test-backup"
-  environment_name = "dev"
+  environment_name = "test"
 
-  source_account_id      = data.aws_arn.source_terraform_role.account
-  destination_account_id = data.aws_caller_identity.current.account_id
+  source_account_id      = var.source_account_id
+  destination_account_id = var.assume_account
 }

terraform/backup-infrastructure/test/main.tf

@@ -1,14 +1,32 @@
-#terraform {
-#  backend "s3" {
-#    bucket         = "project-env-backup-tf-bucket"  # change this to the destination account terraform state s3 bucket name
-#    key            = "project-env-backup.tfstate"  # change this to the destination account terraform state s3 key name
-#    dynamodb_table = "project-env-backup-lock-table"  # change this to the destination account terraform state dynamodb table name
-#    region = "eu-west-2"
-#  }
-#}
-
-
 provider "aws" {
-  alias  = "source"
   region = "eu-west-2"
+
+  assume_role {
+    role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}"
+  }
+
+  default_tags {
+    tags = {
+      project_name = local.project_name
+      workspace    = terraform.workspace
+    }
+  }
+}
+
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.76.0"
+    }
+  }
+
+  backend "s3" {
+    region               = "eu-west-2"
+    bucket               = "nhsd-nrlf--terraform-state"
+    dynamodb_table       = "nhsd-nrlf--terraform-state-lock"
+    key                  = "terraform-state-dev-backup-infrastructure"
+    workspace_key_prefix = "nhsd-nrlf"
+    encrypt              = false
+  }
 }

terraform/backup-infrastructure/test/vars.tf

@@ -1,4 +1,15 @@
-variable "source_terraform_role_arn" {
-  description = "ARN of the terraform role in the source account"
+variable "assume_account" {
+  description = "The account id to deploy the infrastructure to"
+  sensitive   = true
+}
+
+variable "assume_role" {
+  description = "Name of the role to assume to deploy the infrastructure"
+  type        = string
+}
+
+variable "source_account_id" {
+  description = "The account id of the backup source account"
   type        = string
+  sensitive   = true
 }