Merge pull request #517 from NHSDigital/release/2025-02-26

Release/2025 02 26

Author: jameslinnell

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     rev: v1.4.0
     hooks:
       - id: detect-secrets
-        exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|src/etl/sds/tests/changelog|src/etl/sds/worker/bulk/transform_bulk/tests|src/etl/sds/worker/bulk/tests/stage_data|src/api/tests/smoke_tests/test_smoke.py|archived_epr/src_old/api/tests/smoke_tests/test_smoke.py"
+        exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|archived_epr/src_old/etl/sds/tests/changelog|archived_epr/src_old/etl/sds/worker/bulk/transform_bulk/tests|archived_epr/src_old/etl/sds/worker/bulk/tests/stage_data|src/api/tests/smoke_tests/test_smoke.py|archived_epr/src_old/api/tests/smoke_tests/test_smoke.py"
 
   - repo: https://github.com/prettier/pre-commit
     rev: 57f39166b5a5a504d6808b87ab98d41ebf095b46

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 2025-02-26
+- [PI-793] Remove EPR ETL
+- [PI-795] Remove EPR repository layers
+- [PI-834] Remove EPR domain logic
+- Dependabot: Update black
+- Dependabot: Update attrs
+
 ## 2025-02-24
 - [PI-794] Remove EPR S3 tests
 - [PI-788] Create Product Search and Delete Flows for test UI

VERSION

@@ -1 +1 @@
-2025.02.24
+2025.02.26

archived_epr/infrastructure/terraform/per_workspace/.terraform-version

@@ -0,0 +1 @@
+1.5.7

archived_epr/infrastructure/terraform/per_workspace/locals.tf

@@ -0,0 +1,15 @@
+locals {
+  region         = "eu-west-2"
+  project        = "nhse-cpm"
+  current_time   = timestamp()
+  workspace_type = var.workspace_type
+  permission_resource_map = {
+    kms      = ["*"]
+    dynamodb = ["${module.eprtable.dynamodb_table_arn}", "${module.eprtable.dynamodb_table_arn}/*", "${module.cpmtable.dynamodb_table_arn}", "${module.cpmtable.dynamodb_table_arn}/*"]
+  }
+  # e.g. api.cpm.dev.national.nhs.uk
+  zone = var.domain
+
+  domain              = "${terraform.workspace}.${var.domain}"
+  etl_snapshot_bucket = contains(["int", "prod"], var.environment) ? "${local.project}--${replace(var.environment, "_", "-")}--snapshot" : "snapshot_not_required"
+}

archived_epr/infrastructure/terraform/per_workspace/main.tf

@@ -0,0 +1,225 @@
+resource "aws_resourcegroups_group" "resource_group" {
+  name        = "${local.project}--${replace(terraform.workspace, "_", "-")}--resource-group"
+  description = "${local.workspace_type} workspace resource group."
+  tags = {
+    Name           = "${local.project}--${replace(terraform.workspace, "_", "-")}--resource-group"
+    CreatedOn      = var.updated_date
+    LastUpdated    = var.updated_date
+    ExpirationDate = var.expiration_date
+  }
+
+  lifecycle {
+    ignore_changes = [tags["CreatedOn"]]
+  }
+
+  resource_query {
+    query = <<JSON
+{
+  "ResourceTypeFilters": ["AWS::AllSupported"],
+  "TagFilters": [
+    {
+      "Key": "Workspace",
+      "Values": ["${replace(terraform.workspace, "_", "-")}"]
+    }
+  ]
+}
+JSON
+  }
+}
+
+data "aws_secretsmanager_secret" "cpm_apigee_api_key" {
+  name = "${var.environment}-apigee-cpm-apikey"
+}
+
+module "eprtable" {
+  source                      = "./modules/api_storage"
+  name                        = "${local.project}--${replace(terraform.workspace, "_", "-")}--epr-table"
+  environment                 = var.environment
+  deletion_protection_enabled = var.deletion_protection_enabled
+  kms_deletion_window_in_days = 7
+  range_key                   = "sk"
+  hash_key                    = "pk"
+
+  attributes = [
+    { name = "pk", type = "S" },
+    { name = "sk", type = "S" },
+    { name = "pk_read", type = "S" },
+    { name = "sk_read", type = "S" },
+  ]
+
+  global_secondary_indexes = [
+    {
+      name            = "idx_gsi_read"
+      hash_key        = "pk_read"
+      range_key       = "sk_read"
+      projection_type = "ALL"
+    }
+  ]
+}
+
+module "cpmtable" {
+  source                      = "./modules/api_storage"
+  name                        = "${local.project}--${replace(terraform.workspace, "_", "-")}--cpm-table"
+  environment                 = var.environment
+  deletion_protection_enabled = var.deletion_protection_enabled
+  kms_deletion_window_in_days = 7
+  range_key                   = "sk"
+  hash_key                    = "pk"
+
+  attributes = [
+    { name = "pk", type = "S" },
+    { name = "sk", type = "S" },
+    { name = "pk_read_1", type = "S" },
+    { name = "sk_read_1", type = "S" },
+    { name = "pk_read_2", type = "S" },
+    { name = "sk_read_2", type = "S" },
+  ]
+
+  global_secondary_indexes = [
+    {
+      name            = "idx_gsi_read_1"
+      hash_key        = "pk_read_1"
+      range_key       = "sk_read_1"
+      projection_type = "ALL"
+    },
+    {
+      name            = "idx_gsi_read_2"
+      hash_key        = "pk_read_2"
+      range_key       = "sk_read_2"
+      projection_type = "ALL"
+    }
+  ]
+}
+
+module "layers" {
+  for_each       = toset(var.layers)
+  source         = "./modules/api_worker/api_layer"
+  name           = each.key
+  python_version = var.python_version
+  layer_name     = "${local.project}--${replace(terraform.workspace, "_", "-")}--${replace(each.key, "_", "-")}"
+  source_path    = "${path.module}/../../../src/layers/${each.key}/dist/${each.key}.zip"
+}
+
+module "third_party_layers" {
+  for_each       = toset(var.third_party_layers)
+  source         = "./modules/api_worker/api_layer"
+  name           = each.key
+  python_version = var.python_version
+  layer_name     = "${local.project}--${replace(terraform.workspace, "_", "-")}--${replace(each.key, "_", "-")}"
+  source_path    = "${path.module}/../../../src/layers/third_party/dist/${each.key}.zip"
+}
+
+module "lambdas" {
+  for_each       = setsubtract(var.lambdas, ["authoriser"])
+  source         = "./modules/api_worker/api_lambda"
+  python_version = var.python_version
+  name           = each.key
+  lambda_name    = "${local.project}--${replace(terraform.workspace, "_", "-")}--${replace(replace(replace(replace(each.key, "_", "-"), "DeviceReferenceData", "DeviceRefData"), "MessageHandlingSystem", "MHS"), "MessageSet", "MsgSet")}"
+
+  //Compact will remove all nulls from the list and create a new one - this is because TF throws an error if there is a null item in the list.
+  layers = concat(
+    compact([for instance in module.layers : contains(var.api_lambda_layers, instance.name) ? instance.layer_arn : null]),
+    [element([for instance in module.third_party_layers : instance if instance.name == "third_party_core"], 0).layer_arn]
+  )
+  source_path = "${path.module}/../../../src/api/${each.key}/dist/${each.key}.zip"
+  allowed_triggers = {
+    "AllowExecutionFromAPIGateway-${replace(terraform.workspace, "_", "-")}--${replace(each.key, "_", "-")}" = {
+      service    = "apigateway"
+      source_arn = "${module.api_entrypoint.execution_arn}/*/*/*"
+    }
+  }
+  environment_variables = {
+    DYNAMODB_TABLE = contains(["createProductTeam", "readProductTeam", "createCpmProduct", "readCpmProduct", "searchCpmProduct", "deleteCpmProduct"], each.key) ? module.cpmtable.dynamodb_table_name : module.eprtable.dynamodb_table_name
+  }
+  attach_policy_statements = length((fileset("${path.module}/../../../src/api/${each.key}/policies", "*.json"))) > 0
+  policy_statements = {
+    for file in fileset("${path.module}/../../../src/api/${each.key}/policies", "*.json") : replace(file, ".json", "") => {
+      effect    = "Allow"
+      actions   = jsondecode(file("${path.module}/../../../src/api/${each.key}/policies/${file}"))
+      resources = local.permission_resource_map[replace(file, ".json", "")]
+    }
+  }
+  memory_size = var.lambda_memory_size
+}
+
+module "authoriser" {
+  name           = "authoriser"
+  source         = "./modules/api_worker/api_lambda"
+  python_version = var.python_version
+  lambda_name    = "${local.project}--${replace(terraform.workspace, "_", "-")}--authoriser"
+  source_path    = "${path.module}/../../../src/api/authoriser/dist/authoriser.zip"
+  environment_variables = {
+    ENVIRONMENT = var.environment
+  }
+  layers = concat(
+    compact([for instance in module.layers : contains(var.api_lambda_layers, instance.name) ? instance.layer_arn : null]),
+    [element([for instance in module.third_party_layers : instance if instance.name == "third_party_core"], 0).layer_arn]
+  )
+  trusted_entities = [
+    {
+      type = "Service",
+      identifiers = [
+        "apigateway.amazonaws.com"
+      ]
+    }
+  ]
+
+  attach_policy_json = true
+  policy_json        = <<-EOT
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Action": "lambda:InvokeFunction",
+              "Effect": "Allow",
+              "Resource": "arn:aws:lambda:eu-west-2:${var.assume_account}:function:${local.project}--${replace(terraform.workspace, "_", "-")}--authoriser"
+          },
+          {
+              "Action": "secretsmanager:GetSecretValue",
+              "Effect": "Allow",
+              "Resource": "${data.aws_secretsmanager_secret.cpm_apigee_api_key.arn}"
+          }
+      ]
+    }
+  EOT
+}
+
+module "domain" {
+  source = "./modules/domain"
+  domain = local.domain
+  zone   = local.zone
+}
+
+module "api_entrypoint" {
+  source              = "./modules/api_entrypoint"
+  assume_account      = var.assume_account
+  project             = local.project
+  name                = "${local.project}--${replace(terraform.workspace, "_", "-")}--api-entrypoint"
+  lambdas             = setsubtract(var.lambdas, ["authoriser"])
+  authoriser_metadata = module.authoriser.metadata
+  domain              = module.domain.domain_cert
+  depends_on          = [module.domain]
+}
+
+data "aws_s3_bucket" "truststore_bucket" {
+  bucket = "${local.project}--${replace(var.environment, "_", "-")}--truststore"
+}
+
+
+module "sds_etl" {
+  source                           = "./modules/etl/sds"
+  workspace_prefix                 = "${local.project}--${replace(terraform.workspace, "_", "-")}"
+  assume_account                   = var.assume_account
+  python_version                   = var.python_version
+  event_layer_arn                  = element([for instance in module.layers : instance if instance.name == "event"], 0).layer_arn
+  third_party_core_layer_arn       = element([for instance in module.third_party_layers : instance if instance.name == "third_party_sds"], 0).layer_arn
+  third_party_sds_update_layer_arn = element([for instance in module.third_party_layers : instance if instance.name == "third_party_sds_update"], 0).layer_arn
+  domain_layer_arn                 = element([for instance in module.layers : instance if instance.name == "domain"], 0).layer_arn
+  sds_layer_arn                    = element([for instance in module.layers : instance if instance.name == "sds"], 0).layer_arn
+  table_name                       = module.eprtable.dynamodb_table_name
+  table_arn                        = module.eprtable.dynamodb_table_arn
+  is_persistent                    = var.workspace_type == "PERSISTENT"
+  truststore_bucket                = data.aws_s3_bucket.truststore_bucket
+  etl_snapshot_bucket              = local.etl_snapshot_bucket
+  environment                      = var.environment
+}

archived_epr/infrastructure/terraform/per_workspace/modules/api_entrypoint/api_gateway/api_gateway.tf

@@ -0,0 +1,122 @@
+resource "aws_api_gateway_rest_api" "api_gateway_rest_api" {
+  name        = var.name
+  description = "API Gateway Rest API - autogenerated from swagger"
+  # UNCOMMENT THIS WHEN ENABLING CUSTOM DOMAINS
+  # disable_execute_api_endpoint = true
+  body = sensitive(local.swagger_file)
+
+  depends_on = [
+    aws_cloudwatch_log_group.api_gateway_access_logs
+  ]
+
+}
+
+resource "aws_api_gateway_deployment" "api_gateway_deployment" {
+  rest_api_id = aws_api_gateway_rest_api.api_gateway_rest_api.id
+
+  triggers = {
+    redeployment    = sha1(jsonencode(aws_api_gateway_rest_api.api_gateway_rest_api.body))
+    resource_change = "${md5(file("${path.module}/api_gateway.tf"))}"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [
+    aws_api_gateway_rest_api.api_gateway_rest_api,
+    aws_cloudwatch_log_group.api_gateway_access_logs,
+    aws_cloudwatch_log_group.api_gateway_execution_logs
+  ]
+}
+
+resource "aws_api_gateway_stage" "api_gateway_stage" {
+  deployment_id        = aws_api_gateway_deployment.api_gateway_deployment.id
+  rest_api_id          = aws_api_gateway_rest_api.api_gateway_rest_api.id
+  stage_name           = "production"
+  xray_tracing_enabled = true
+
+  access_log_settings {
+    destination_arn = aws_cloudwatch_log_group.api_gateway_access_logs.arn
+    format = jsonencode({
+      requestid : "$context.requestId",
+      ip : "$context.identity.sourceIp",
+      user_agent : "$context.identity.userAgent",
+      request_time : "$context.requestTime",
+      http_method : "$context.httpMethod",
+      path : "$context.path",
+      status : "$context.status",
+      protocol : "$context.protocol",
+      response_length : "$context.responseLength",
+      x_correlationid : "$context.authorizer.x-correlation-id",
+      nhsd_correlationid : "$context.authorizer.nhsd-correlation-id"
+      environment : terraform.workspace
+    })
+  }
+
+  depends_on = [
+    aws_api_gateway_deployment.api_gateway_deployment,
+    aws_api_gateway_rest_api.api_gateway_rest_api,
+    aws_cloudwatch_log_group.api_gateway_access_logs,
+    aws_cloudwatch_log_group.api_gateway_execution_logs
+  ]
+}
+
+resource "aws_api_gateway_method_settings" "api_gateway_method_settings" {
+  rest_api_id = aws_api_gateway_rest_api.api_gateway_rest_api.id
+  stage_name  = aws_api_gateway_stage.api_gateway_stage.stage_name
+  method_path = "*/*"
+  settings {
+    logging_level      = "INFO"
+    data_trace_enabled = true
+  }
+
+  depends_on = [
+    aws_api_gateway_rest_api.api_gateway_rest_api,
+    aws_api_gateway_stage.api_gateway_stage
+  ]
+}
+
+resource "aws_api_gateway_gateway_response" "api_access_denied" {
+  rest_api_id   = aws_api_gateway_rest_api.api_gateway_rest_api.id
+  response_type = "ACCESS_DENIED"
+  response_templates = {
+    "application/json" = jsonencode({
+      errors : [{
+        code : "PROCESSING"
+        message : "$context.authorizer.error"
+      }]
+    })
+  }
+  response_parameters = {
+    "gatewayresponse.header.Access-Control-Allow-Origin" = "'*'"
+  }
+}
+
+resource "aws_api_gateway_gateway_response" "api_default_4xx" {
+  rest_api_id   = aws_api_gateway_rest_api.api_gateway_rest_api.id
+  response_type = "DEFAULT_4XX"
+  response_templates = {
+    "application/json" = jsonencode({
+      errors : [{
+        code : "PROCESSING"
+        message : "$context.error.message"
+      }]
+  }) }
+  response_parameters = { "gatewayresponse.header.Access-Control-Allow-Origin" = "'*'"
+  }
+}
+
+resource "aws_api_gateway_gateway_response" "api_default_5xx" {
+  rest_api_id   = aws_api_gateway_rest_api.api_gateway_rest_api.id
+  response_type = "DEFAULT_5XX"
+  response_templates = {
+    "application/json" = jsonencode({
+      errors : [{
+        code : "PROCESSING"
+        message : "exception"
+      }]
+  }) }
+  response_parameters = { "gatewayresponse.header.Access-Control-Allow-Origin" = "'*'"
+  }
+}