CCM-12607 add letter pdf TTL functionality (#67)

* CCM-12607: add ttl-poll-lambda code and utils

* CCM-12607: add ttl-create-lambda code

* CCM-12607: ttl-poll-lambda linted

* CCM-12607: terraform bits

* CCM-12607: remove write shards from poller

* CCM-12607: move ttl terraform into dl

* CCM-12607: enable point in time recovery for the table

* CCM-12607: sonar config tweaks

* CCM-12607: rename test folders

* CCM-12607: sonar config tweaks

* CCM-12607: sonar config tweaks

* CCM-12607: sonar config tweaks

* CCM-12607: correction for log streaming destination address

* CCM-12607: Devcontainer tweaks

* CCM-12607: Enable typecheck and linting in CI workflow

* CCM-12607: Fix linting errors

* CCM-12607: Address TF review comments

* CCM-12607: Address ttl-create-lambda review comments

* CCM-12607: Update ttl-create-lambda package name

* CCM-12607: ttl-poll-lambda review changes

* CCM-12607: Added additional unit tests

* CCM-12607: Address Sonar issues

* CCM-12607: Fix one more Sonar issue

* CCM-12607: Tweak ttl-expiry-service test

---------

Co-authored-by: Gareth Allan <[email protected]>

Author: Ian-Hodges

.devcontainer/old/devcontainer.json

@@ -23,6 +23,7 @@
         "github.remotehub",
         "github.vscode-github-actions",
         "github.vscode-pull-request-github",
+        "hashicorp.terraform",
         "hediet.vscode-drawio",
         "johnpapa.vscode-peacock",
         "joshx.workspace-terminals",
@@ -91,6 +92,7 @@
   "mounts": [
     "source=${localEnv:HOME}/.gnupg,target=/home/vscode/.gnupg,type=bind,consistency=cached",
     "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached",
+    "source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached",
     "source=/usr/local/share/ca-certificates,target=/home/ca-certificates,type=bind,consistency=cached"
   ],
   "name": "NHS Notify Digital Letters",

eslint.config.mjs

@@ -77,7 +77,6 @@ export default defineConfig([
       'import-x/resolver-next': [
         eslintImportResolverTypescript.createTypeScriptImportResolver({
           project: [
-            'frontend/tsconfig.json',
             'lambdas/*/tsconfig.json',
             'tests/test-team/tsconfig.json',
             'utils/*/tsconfig.json',
@@ -140,7 +139,7 @@ export default defineConfig([
 
   // prettier
   prettierRecommended,
-  { rules: { ...prettierConfigRules, 'prettier/prettier': 2 } },
+  { rules: { ...prettierConfigRules, 'prettier/prettier': ['error', { singleQuote: true }] } },
 
   // jsxA11y
   {
@@ -169,20 +168,6 @@ export default defineConfig([
     plugins: { html },
   },
 
-  // Next.js
-  ...compat.config({
-    extends: ['next', 'next/core-web-vitals', 'next/typescript'],
-    settings: {
-      next: {
-        rootDir: 'frontend',
-      },
-    },
-    rules: {
-      // needed because next lint rules look for a pages directory
-      '@next/next/no-html-link-for-pages': 0,
-    },
-  }),
-
   // json
   {
     files: ['**/*.json'],
@@ -243,6 +228,12 @@ export default defineConfig([
       'no-relative-import-paths/no-relative-import-paths': 2,
     },
   },
+  {
+    files: ['utils/**'],
+    rules: {
+      'no-relative-import-paths/no-relative-import-paths': 0,
+    },
+  },
   {
     files: ['scripts/**'],
     rules: {

infrastructure/terraform/components/.gitkeep

@@ -11,6 +11,7 @@ No requirements.
 |------|-------------|------|---------|:--------:|
 | <a name="input_aws_account_id"></a> [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
+| <a name="input_enable_dynamodb_delete_protection"></a> [enable\_dynamodb\_delete\_protection](#input\_enable\_dynamodb\_delete\_protection) | Enable DynamoDB Delete Protection on all Tables | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_force_lambda_code_deploy"></a> [force\_lambda\_code\_deploy](#input\_force\_lambda\_code\_deploy) | If the lambda package in s3 has the same commit id tag as the terraform build branch, the lambda will not update automatically. Set to True if making changes to Lambda code from on the same commit for example during development | `bool` | `false` | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
@@ -20,14 +21,21 @@ No requirements.
 | <a name="input_mesh_poll_schedule"></a> [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"cron(0,30 8-16 ? * MON-FRI *)"` | no |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes |
+| <a name="input_queue_batch_size"></a> [queue\_batch\_size](#input\_queue\_batch\_size) | maximum number of queue items to process | `number` | `10` | no |
+| <a name="input_queue_batch_window_seconds"></a> [queue\_batch\_window\_seconds](#input\_queue\_batch\_window\_seconds) | maximum time in seconds between processing events | `number` | `10` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS Region | `string` | n/a | yes |
+| <a name="input_shared_infra_account_id"></a> [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Shared Infra Account ID (numeric) | `string` | n/a | yes |
+| <a name="input_ttl_poll_schedule"></a> [ttl\_poll\_schedule](#input\_ttl\_poll\_schedule) | Schedule to poll for any overdue TTL records | `string` | `"rate(10 minutes)"` | no |
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-kms.zip | n/a |
+| <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-kms.zip | n/a |
 | <a name="module_mesh_poll"></a> [mesh\_poll](#module\_mesh\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
 | <a name="module_s3bucket_letters"></a> [s3bucket\_letters](#module\_s3bucket\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
+| <a name="module_sqs_ttl"></a> [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
+| <a name="module_ttl_create"></a> [ttl\_create](#module\_ttl\_create) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
+| <a name="module_ttl_poll"></a> [ttl\_poll](#module\_ttl\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
 ## Outputs
 
 No outputs.

infrastructure/terraform/components/dl/README.md

@@ -0,0 +1,54 @@
+resource "aws_dynamodb_table" "ttl" {
+  name         = "${local.csi}-ttl"
+  billing_mode = "PAY_PER_REQUEST"
+  hash_key     = "PK"
+  range_key    = "SK"
+
+  deletion_protection_enabled = var.enable_dynamodb_delete_protection
+
+  attribute {
+    name = "PK"
+    type = "S"
+  }
+
+  attribute {
+    name = "SK"
+    type = "S"
+  }
+
+  attribute {
+    name = "dateOfExpiry"
+    type = "S"
+  }
+
+  attribute {
+    name = "ttl"
+    type = "N"
+  }
+
+  ttl {
+    enabled        = true
+    attribute_name = "ttl"
+  }
+
+  point_in_time_recovery {
+    enabled = true
+  }
+
+  global_secondary_index {
+    name            = "dateOfExpiryIndex"
+    hash_key        = "dateOfExpiry"
+    projection_type = "ALL"
+    range_key       = "ttl"
+  }
+
+  server_side_encryption {
+    enabled     = true
+    kms_key_arn = module.kms.key_arn
+  }
+
+  stream_enabled   = true
+  stream_view_type = "OLD_IMAGE"
+
+  tags = local.default_tags
+}

infrastructure/terraform/components/dl/dynamodb_table_ttl.tf

@@ -0,0 +1,9 @@
+resource "aws_lambda_event_source_mapping" "ttl_create_lambda" {
+  event_source_arn                   = module.sqs_ttl.sqs_queue_arn
+  function_name                      = module.ttl_create.function_name
+  batch_size                         = var.queue_batch_size
+  maximum_batching_window_in_seconds = var.queue_batch_window_seconds
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+}

infrastructure/terraform/components/dl/lambda_event_source_mapping_ttl_create_lambda.tf

@@ -1,3 +1,6 @@
 locals {
   aws_lambda_functions_dir_path = "../../../../lambdas"
+  log_destination_arn           = "arn:aws:logs:${var.region}:${var.shared_infra_account_id}:destination:nhs-main-obs-firehose-logs"
+
+  ttl_shard_count = 3
 }

infrastructure/terraform/components/dl/locals.tf

@@ -1,5 +1,5 @@
 module "kms" {
-  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-kms.zip"
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-kms.zip"
 
 
   providers = {
@@ -70,10 +70,12 @@ data "aws_iam_policy_document" "kms" {
       test     = "ArnLike"
       variable = "kms:EncryptionContext:aws:sqs:arn"
       values = [
-        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-*",
+        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-ttl-queue",
+        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-ttl-dlq",
       ]
     }
   }
+
   statement {
     sid    = "AllowCloudWatchLogsEncrypt"
     effect = "Allow"
@@ -97,4 +99,24 @@ data "aws_iam_policy_document" "kms" {
       "*",
     ]
   }
+
+  statement {
+    sid    = "AllowDynamoDBEncryption"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["dynamodb.amazonaws.com"]
+    }
+
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Describe*"
+    ]
+
+    resources = ["*"]
+  }
 }

infrastructure/terraform/components/dl/module_kms.tf

@@ -32,6 +32,10 @@ module "mesh_poll" {
   force_lambda_code_deploy = var.force_lambda_code_deploy
   enable_lambda_insights   = false
 
+  send_to_firehose          = true
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
   lambda_env_vars = {
   }
 }

infrastructure/terraform/components/dl/module_lambda_mesh_poll.tf

@@ -0,0 +1,88 @@
+module "ttl_create" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip"
+
+  function_name = "ttl-create"
+  description   = "A function for creating TTL records"
+
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  group          = var.group
+
+  log_retention_in_days = var.log_retention_in_days
+  kms_key_arn           = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.ttl_create_lambda.json
+  }
+
+  function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
+  function_code_base_path = local.aws_lambda_functions_dir_path
+  function_code_dir       = "ttl-create-lambda/dist"
+  function_include_common = true
+  handler_function_name   = "handler"
+  runtime                 = "nodejs22.x"
+  memory                  = 128
+  timeout                 = 60
+  log_level               = var.log_level
+
+  force_lambda_code_deploy = var.force_lambda_code_deploy
+  enable_lambda_insights   = false
+
+  send_to_firehose          = true
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
+  lambda_env_vars = {
+    "TTL_TABLE_NAME"      = aws_dynamodb_table.ttl.name
+    "TTL_WAIT_TIME_HOURS" = 24
+    "TTL_SHARD_COUNT"     = local.ttl_shard_count
+  }
+}
+
+data "aws_iam_policy_document" "ttl_create_lambda" {
+  statement {
+    sid    = "AllowTtlDynamoAccess"
+    effect = "Allow"
+
+    actions = [
+      "dynamodb:PutItem",
+    ]
+
+    resources = [
+      aws_dynamodb_table.ttl.arn,
+    ]
+  }
+
+  statement {
+    sid    = "KMSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+
+  statement {
+    sid    = "SQSPermissionsTtlQueue"
+    effect = "Allow"
+
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:GetQueueUrl"
+    ]
+
+    resources = [
+      module.sqs_ttl.sqs_queue_arn,
+    ]
+  }
+}