Merge pull request #1033 from NHSDigital/feature/made14-NRL-1591-prod-infra-fixes

[NRL-1591] Add prod backup infrastructure code

Author: mattdean3-nhs

terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql

@@ -33,12 +33,12 @@ WITH
    , event_function_request_id
    , event_correlation_id
    , event_xray_trace_id
-   , event_pointer_types
+   --, event_pointer_types
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_deletedocumentreference
 )
-, pr AS (
+/*, pr AS (
    SELECT
      time
    , event_timestamp
@@ -56,8 +56,8 @@ WITH
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_readdocumentreference
-)
-, ps AS (
+)*/
+/*, ps AS (
    SELECT
      time
    , event_timestamp
@@ -75,7 +75,7 @@ WITH
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_searchdocumentreference
-)
+)*/
 , psp AS (
    SELECT
      time
@@ -90,12 +90,12 @@ WITH
    , event_function_request_id
    , event_correlation_id
    , event_xray_trace_id
-   , event_pointer_types
+   --, event_pointer_types
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_searchpostdocumentreference
 )
-, pu AS (
+/*, pu AS (
    SELECT
      time
    , event_timestamp
@@ -113,7 +113,7 @@ WITH
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_updatedocumentreference
-)
+)*/
 , pus AS (
    SELECT
      time
@@ -128,7 +128,7 @@ WITH
    , event_function_request_id
    , event_correlation_id
    , event_xray_trace_id
-   , event_pointer_types
+   --, event_pointer_types
    , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
    FROM
      producer_upsertdocumentreference
@@ -140,18 +140,18 @@ WITH
 UNION    SELECT *
    FROM
      pd
-UNION    SELECT *
+/*UNION    SELECT *
    FROM
      pr
 UNION    SELECT *
    FROM
-     ps
+     ps*/
 UNION    SELECT *
    FROM
      psp
-UNION    SELECT *
+/*UNION    SELECT *
    FROM
-     pu
+     pu*/
 UNION    SELECT *
    FROM
      pus
@@ -177,7 +177,7 @@ SELECT
 , event_function_request_id
 , b.event_correlation_id
 , b.event_xray_trace_id
-, event_pointer_types
+--, event_pointer_types
 , oc.user_ods
 FROM
   (base b

terraform/account-wide-infrastructure/modules/glue/glue.tf

@@ -90,7 +90,8 @@ resource "aws_glue_job" "glue_job" {
   description       = "Transfer logs from source to bucket"
   glue_version      = "5.0"
   worker_type       = "G.1X"
-  timeout           = 2880
+  execution_class   = "STANDARD"
+  timeout           = 60 # minutes
   max_retries       = 0
   number_of_workers = 4
   command {
@@ -110,5 +111,7 @@ resource "aws_glue_job" "glue_job" {
     "--enable-continuous-log-filter"    = "true"
     "--enable-metrics"                  = "true"
     "--extra-py-files"                  = "s3://${aws_s3_bucket.code-bucket.id}/src.zip"
+    "--enable-job-insights"             = "true"
+    "--job-language"                    = "python"
   }
 }

terraform/account-wide-infrastructure/prod/aws-backup.tf

@@ -174,7 +174,7 @@ module "source" {
         "schedule" : "cron(30 0 ? * 4)" # every Thursday from 00:30 to precede releases
         "copy_action" : [{
           "cold_storage_after" : 14 # ensures 2 warm including one from previous release
-          "delete_after" : 105
+          "delete_after" : 15 * 7
         }],
         "lifecycle" : {
           "delete_after" : 2

terraform/account-wide-infrastructure/test/main.tf

@@ -12,6 +12,13 @@ provider "aws" {
   }
 
 }
+provider "awscc" {
+  region = local.region
+
+  assume_role = {
+    role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}"
+  }
+}
 
 provider "awscc" {
   region = local.region

terraform/backup-infrastructure/README.md

@@ -28,8 +28,8 @@ Then, initialise the terraform backup workspace. For the test account:
 ```shell
 $ cd test
 $ terraform init && ( \
-    terraform workspace new backup-infra-test || \
-    terraform workspace select backup-infra-test )
+    terraform workspace new test || \
+    terraform workspace select test )
 ```
 
 If you want to apply changes to prod, use the `prod` directory and the `backup-infra-prod` terraform workspace.
@@ -69,8 +69,8 @@ Then, initialise your terraform workspace. For the test account:
 ```shell
 $ cd test
 $ terraform init && ( \
-    terraform workspace new backup-infra-test || \
-    terraform workspace select backup-infra-test )
+    terraform workspace new test || \
+    terraform workspace select test )
 ```
 
 If you want to destroy resources in prod, use the `prod` directory and the `backup-infra-prod` terraform workspace.

terraform/backup-infrastructure/prod/aws-backup.tf

@@ -0,0 +1,42 @@
+
+# We need a key for the backup vaults. This key will be used to encrypt the backups themselves.
+# We need one per vault (on the assumption that each vault will be in a different account).
+resource "aws_kms_key" "destination_backup_key" {
+  description             = "KMS key for AWS Backup vaults"
+  deletion_window_in_days = 7
+  enable_key_rotation     = true
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Sid    = "Enable IAM User Permissions"
+        Principal = {
+          AWS = "arn:aws:iam::${local.destination_account_id}:root"
+        }
+        Action   = "kms:*"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+module "destination" {
+  source = "../modules/aws-backup-destination"
+
+  source_account_name     = "prod" # please note that the assigned value would be the prefix in aws_backup_vault.vault.name
+  account_id              = local.destination_account_id
+  source_account_id       = local.source_account_id
+  kms_key                 = aws_kms_key.destination_backup_key.arn
+  enable_vault_protection = false
+}
+
+###
+# Destination vault ARN output
+###
+
+output "destination_vault_arn" {
+  # The ARN of the backup vault in the destination account is needed by
+  # the source account to copy backups into it.
+  value = module.destination.vault_arn
+}

terraform/backup-infrastructure/prod/data.tf

@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}

terraform/backup-infrastructure/prod/locals.tf

@@ -0,0 +1,8 @@
+locals {
+  # Adjust these as required
+  project_name     = "nrlf-prod-backup"
+  environment_name = "prod"
+
+  source_account_id      = var.source_account_id
+  destination_account_id = var.assume_account
+}

terraform/backup-infrastructure/prod/main.tf

@@ -0,0 +1,32 @@
+provider "aws" {
+  region = "eu-west-2"
+
+  assume_role {
+    role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}"
+  }
+
+  default_tags {
+    tags = {
+      project_name = local.project_name
+      workspace    = terraform.workspace
+    }
+  }
+}
+
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.76.0"
+    }
+  }
+
+  backend "s3" {
+    region               = "eu-west-2"
+    bucket               = "nhsd-nrlf--terraform-state"
+    dynamodb_table       = "nhsd-nrlf--terraform-state-lock"
+    key                  = "terraform-state-backup-infrastructure"
+    workspace_key_prefix = "nhsd-nrlf"
+    encrypt              = false
+  }
+}

terraform/backup-infrastructure/prod/vars.tf

@@ -0,0 +1,15 @@
+variable "assume_account" {
+  description = "The account id to deploy the infrastructure to"
+  sensitive   = true
+}
+
+variable "assume_role" {
+  description = "Name of the role to assume to deploy the infrastructure"
+  type        = string
+}
+
+variable "source_account_id" {
+  description = "The account id of the backup source account"
+  type        = string
+  sensitive   = true
+}