[NRL-1860] Add account-wide meta-data bucket module. Add metadata bucket for perf-test env

Author: mattdean3-nhs

scripts/seed_nft_tables.py

@@ -174,6 +174,7 @@ def _populate_seed_table(
     )
     doc_ref_counter = 0
     batch_counter = 0
+    unprocessed_count = 0
 
     pointer_data: list[list[str]] = []
 
@@ -189,8 +190,8 @@ def _populate_seed_table(
             )
 
             if response.get("UnprocessedItems"):
-                logger.error(
-                    f"Unprocessed items in batch write: {len(response.get('UnprocessedItems'))}"
+                unprocessed_count += len(
+                    response.get("UnprocessedItems").get(table_name, [])
                 )
 
             batch_upsert_items = []
@@ -221,13 +222,13 @@ def _populate_seed_table(
         if px_counter % 1000 == 0:
             print(".", end="", flush=True)
         if px_counter % 100000 == 0:
-            print(f" {px_counter} patients processed")
+            print(f" {px_counter} patients processed ({doc_ref_counter} pointers).")
 
     print(" Done.")
 
     end_time = datetime.now(tz=timezone.utc)
     print(
-        f"Created {doc_ref_counter} pointers in {timedelta.total_seconds(end_time - start_time)} seconds."
+        f"Created {doc_ref_counter} pointers in {timedelta.total_seconds(end_time - start_time)} seconds (unprocessed: {unprocessed_count})."
     )
 
     with open("./dist/seed-nft-pointers.csv", "w") as f:

terraform/account-wide-infrastructure/modules/metadata-bucket/output.tf

@@ -0,0 +1,4 @@
+output "bucket_name" {
+  description = "Name of the metadata S3 bucket"
+  value       = aws_s3_bucket.metadata_bucket.bucket
+}

terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf

@@ -0,0 +1,56 @@
+resource "aws_s3_bucket" "metadata_bucket" {
+  bucket        = "${var.name_prefix}-metadata"
+  force_destroy = false
+}
+
+resource "aws_s3_bucket_policy" "metadata_bucket_policy" {
+  bucket = aws_s3_bucket.metadata_bucket.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Id      = "metadata_bucket_policy"
+    Statement = [
+      {
+        Sid       = "HTTPSOnly"
+        Effect    = "Deny"
+        Principal = "*"
+        Action    = "s3:*"
+        Resource = [
+          aws_s3_bucket.metadata_bucket.arn,
+          "${aws_s3_bucket.metadata_bucket.arn}/*",
+        ]
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
+      },
+    ]
+  })
+}
+
+resource "aws_s3_bucket_public_access_block" "metadata_bucket_public_access_block" {
+  bucket = aws_s3_bucket.metadata_bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "metadata_bucket" {
+  bucket = aws_s3_bucket.metadata_bucket.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_versioning" "metadata_bucket" {
+  bucket = aws_s3_bucket.metadata_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}

terraform/account-wide-infrastructure/modules/metadata-bucket/vars.tf

@@ -0,0 +1,4 @@
+variable "name_prefix" {
+  type        = string
+  description = "The prefix to apply to all resources in the module."
+}

terraform/account-wide-infrastructure/test/s3.tf

@@ -64,3 +64,8 @@ module "perftest-truststore-bucket" {
   name_prefix             = "nhsd-nrlf--perftest"
   server_certificate_file = "../../../truststore/server/perftest.pem"
 }
+
+module "perftest-metadata-bucket" {
+  source      = "../modules/metadata-bucket"
+  name_prefix = "nhsd-nrlf--perftest"
+}