CCM-12833: Without AgentMD - DO NOT MERGE

Author: harrim91

infrastructure/terraform/modules/backend-api/module_lambda_data_export_csv_writer.tf

@@ -0,0 +1,106 @@
+module "lambda_data_export_csv_writer" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.22/terraform-lambda.zip"
+
+  project        = var.project
+  environment    = var.environment
+  component      = var.component
+  aws_account_id = var.aws_account_id
+  region         = var.region
+
+  kms_key_arn = var.kms_key_arn
+
+  function_name          = "data-export-csv-writer"
+  function_module_name   = "data-export-csv-writer"
+  handler_function_name  = "handler"
+  description            = "Lambda that consumes SQS messages and writes the 'data' object to S3 as CSV"
+  memory                 = 512
+  timeout                = 30
+  runtime                = "nodejs20.x"
+  log_retention_in_days  = var.log_retention_in_days
+  function_s3_bucket     = var.function_s3_bucket
+  function_code_base_path = local.lambdas_dir
+  function_code_dir       = "data-export-csv-writer/dist"
+
+  lambda_env_vars = {
+    BUCKET_NAME = module.s3bucket_data_export.id
+    KEY_PREFIX  = "exports/"
+    NODE_OPTIONS = "--enable-source-maps"
+  }
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.lambda_data_export_csv_writer.json
+  }
+
+  send_to_firehose          = var.send_to_firehose
+  log_destination_arn       = var.log_destination_arn
+  log_subscription_role_arn = var.log_subscription_role_arn
+}
+
+resource "aws_lambda_event_source_mapping" "lambda_data_export_csv_writer" {
+  event_source_arn                   = module.sqs_data_export.sqs_queue_arn
+  function_name                      = module.lambda_data_export_csv_writer.function_name
+  batch_size                         = 10
+  maximum_batching_window_in_seconds = 0
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+
+  scaling_config {
+    maximum_concurrency = 5
+  }
+}
+
+data "aws_iam_policy_document" "lambda_data_export_csv_writer" {
+  statement {
+    sid    = "AllowSQSDLQ"
+    effect = "Allow"
+    actions = [
+      "sqs:SendMessage",
+    ]
+    resources = [
+      module.sqs_data_export.sqs_dlq_arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowSQSConsume"
+    effect = "Allow"
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:ChangeMessageVisibility",
+    ]
+    resources = [
+      module.sqs_data_export.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3Write"
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:AbortMultipartUpload",
+      "s3:ListMultipartUploadParts"
+    ]
+    resources = [
+      "${module.s3bucket_data_export.arn}/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowKMS"
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:Encrypt",
+      "kms:GenerateDataKey*",
+      "kms:ReEncrypt*",
+    ]
+    resources = [
+      var.kms_key_arn,
+    ]
+  }
+}

infrastructure/terraform/modules/backend-api/module_s3bucket_data_export.tf

@@ -0,0 +1,13 @@
+module "s3bucket_data_export" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-s3bucket.zip"
+
+  name = "data-export"
+
+  aws_account_id = var.aws_account_id
+  region         = var.region
+  project        = var.project
+  environment    = var.environment
+  component      = var.component
+
+  kms_key_arn = var.kms_key_arn
+}

infrastructure/terraform/modules/backend-api/module_sqs_data_export_queue.tf

@@ -0,0 +1,16 @@
+module "sqs_data_export" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-sqs.zip"
+
+  aws_account_id = var.aws_account_id
+  component      = var.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  name           = "data-export"
+
+  sqs_kms_key_arn = var.kms_key_arn
+
+  visibility_timeout_seconds = 60
+
+  create_dlq = true
+}

lambdas/data-export-csv-writer/README.md

@@ -0,0 +1,58 @@
+# Data Export CSV Writer Lambda
+
+Consumes messages from the `data-export` SQS queue and writes the `data` object contained in each message body to a CSV file in the `data-export` S3 bucket.
+
+## Message Format
+
+Each SQS message is expected to have a JSON body with a top-level `data` object, for example:
+
+```json
+{
+  "eventType": "example.created",
+  "data": {"id": 123, "name": "Example"},
+  "meta": {"correlationId": "..."}
+}
+``
+
+Only the `data` object is extracted; other fields are ignored.
+
+## CSV Output
+
+For a batch of messages the lambda produces one CSV file whose headers are the union of all keys across the collected `data` objects (first-seen order). A file name like:
+
+```
+exports/2025-11-06T12-00-00-000Z-abcdef12.csv
+```
+
+is written to the S3 bucket defined by the `BUCKET_NAME` environment variable (provisioned as `data-export`).
+
+## Environment Variables
+
+| Name        | Description                               |
+|-------------|-------------------------------------------|
+| BUCKET_NAME | Target S3 bucket for CSV files            |
+| KEY_PREFIX  | Optional key prefix (defaults to exports/) |
+
+## Building
+
+From the repository root after installing workspaces:
+
+```bash
+npm install
+npm run lambda-build --workspace lambdas/data-export-csv-writer
+```
+
+## Testing
+
+```bash
+npm run test:unit --workspace lambdas/data-export-csv-writer
+```
+
+## Deployment
+
+Terraform defines:
+* SQS queue & DLQ: `module.sqs_data_export`
+* S3 bucket: `module.s3bucket_data_export`
+* Lambda + Event Source Mapping: `module.lambda_data_export_csv_writer` + `aws_lambda_event_source_mapping.lambda_data_export_csv_writer`
+
+Granting S3 write & SQS consume permissions is handled in the IAM policy document in `module_lambda_data_export_csv_writer.tf`.

lambdas/data-export-csv-writer/build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+rm -rf dist
+
+npx esbuild \
+    --bundle \
+    --minify \
+    --sourcemap \
+    --target=es2020 \
+    --platform=node \
+    --loader:.node=file \
+    --entry-names=[name] \
+    --outdir=dist \
+    src/data-export-csv-writer.ts

lambdas/data-export-csv-writer/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "nhs-notify-data-export-csv-writer",
+  "private": true,
+  "version": "0.0.1",
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.911.0"
+  },
+  "devDependencies": {
+    "@swc/core": "^1.11.13",
+    "@swc/jest": "^0.2.37",
+    "@tsconfig/node20": "^20.1.5",
+    "@types/aws-lambda": "^8.10.148",
+    "@types/jest": "^29.5.14",
+    "esbuild": "^0.25.9",
+    "jest": "^29.7.0",
+    "typescript": "^5.8.2"
+  },
+  "scripts": {
+    "lambda-build": "./build.sh",
+    "test:unit": "jest",
+    "typecheck": "tsc --noEmit"
+  }
+}

lambdas/data-export-csv-writer/src/__tests__/objectsToCsv.test.ts

@@ -0,0 +1,17 @@
+import { objectsToCsv } from '../data-export-csv-writer';
+
+describe('objectsToCsv', () => {
+  it('creates a csv with union headers and escaped values', () => {
+    const csv = objectsToCsv([
+      { a: 1, b: 'hello' },
+      { b: 'he,llo', c: '"quoted"' }
+    ]);
+
+    expect(csv.split('\n')[0]).toBe('a,b,c');
+    expect(csv).toContain('1,hello,');
+    // he,llo must be quoted
+    expect(csv).toMatch(/"he,llo"/);
+    // quotes escaped
+    expect(csv).toMatch(/""quoted""/);
+  });
+});

lambdas/data-export-csv-writer/src/data-export-csv-writer.ts

@@ -0,0 +1,68 @@
+import { SQSHandler } from 'aws-lambda';
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+
+const s3 = new S3Client({});
+
+// Convert array of objects to CSV string.
+export const objectsToCsv = (rows: Record<string, unknown>[]): string => {
+  if (rows.length === 0) return '';
+
+  // Collect union of keys preserving first-seen order.
+  const headerSet: string[] = [];
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!headerSet.includes(key)) headerSet.push(key);
+    }
+  }
+
+  const escape = (val: unknown): string => {
+    if (val === null || val === undefined) return '';
+    const str = typeof val === 'string' ? val : JSON.stringify(val);
+    const needsQuotes = /[",\n]/.test(str);
+    const escaped = str.replace(/"/g, '""');
+    return needsQuotes ? `"${escaped}"` : escaped;
+  };
+
+  const lines = [headerSet.join(',')];
+  for (const row of rows) {
+    lines.push(headerSet.map((h) => escape((row as any)[h])).join(','));
+  }
+  return lines.join('\n');
+};
+
+export const handler: SQSHandler = async (event) => {
+  const bucket = process.env.BUCKET_NAME;
+  if (!bucket) {
+    throw new Error('BUCKET_NAME env var not set');
+  }
+  const prefix = process.env.KEY_PREFIX ?? '';
+
+  const dataRows: Record<string, unknown>[] = [];
+
+  for (const record of event.Records) {
+    try {
+      const payload: any = JSON.parse(record.body);
+      if (payload && typeof payload === 'object' && payload.data && typeof payload.data === 'object') {
+        dataRows.push(payload.data as Record<string, unknown>);
+      }
+    } catch (err) {
+      console.warn('Skipping record - invalid JSON', err);
+    }
+  }
+
+  if (dataRows.length === 0) return;
+
+  const csv = objectsToCsv(dataRows);
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const random = Math.random().toString(36).substring(2, 10);
+  const key = `${prefix}${timestamp}-${random}.csv`;
+
+  await s3.send(
+    new PutObjectCommand({
+      Bucket: bucket,
+      Key: key,
+      Body: csv,
+      ContentType: 'text/csv'
+    })
+  );
+};

lambdas/data-export-csv-writer/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "resolveJsonModule": true
+  },
+  "extends": "@tsconfig/node20/tsconfig.json",
+  "include": [
+    "src/**/*"
+  ]
+}