CCM-12833: AgentsMD Testing - DO NOT MERGE

Author: jamesthompson26-nhs

agents.md

@@ -0,0 +1,101 @@
+module "queue_csv_writer_lambda" {
+  source                    = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.22/terraform-lambda.zip"
+
+  function_name             = "queue-csv-writer"
+  description               = "Lambda that consumes SQS messages and writes data objects to CSV in S3"
+
+  aws_account_id            = var.aws_account_id
+  component                 = var.component
+  environment               = var.environment
+  project                   = var.project
+  region                    = var.region
+  group                     = var.group
+
+  log_retention_in_days     = var.log_retention_in_days
+  kms_key_arn               = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.queue_csv_writer_lambda.json
+  }
+
+  function_s3_bucket        = local.acct.s3_buckets["artefacts"]["id"]
+  function_code_base_path   = local.lambdas_source_code_dir
+  function_code_dir         = "queue-csv-writer/dist"
+  handler_function_name     = "handler"
+  runtime                   = "nodejs20.x"
+  memory                    = 512  # agent: rationale lightweight CSV transformation; no heavy parsing/compression
+  timeout                   = 20
+
+  lambda_env_vars = {
+    EVENT_CSV_BUCKET_NAME   = module.s3bucket_event_csv.id
+  }
+}
+
+resource "aws_lambda_event_source_mapping" "queue_csv_writer" {
+  event_source_arn                   = module.sqs_event_csv.sqs_queue_arn
+  function_name                      = module.queue_csv_writer_lambda.function_name
+  batch_size                         = 10  # agent: rationale small batch keeps latency low and limits CSV size per object
+  maximum_batching_window_in_seconds = 0
+  function_response_types            = ["ReportBatchItemFailures"]
+}
+
+data "aws_iam_policy_document" "queue_csv_writer_lambda" {
+  statement {
+    sid    = "AllowSQS"
+    effect = "Allow"
+
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:ChangeMessageVisibility",
+    ]
+
+    resources = [
+      module.sqs_event_csv.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowSQSDLQ"
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+    ]
+
+    resources = [
+      module.sqs_event_csv.sqs_dlq_arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3Write"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "${module.s3bucket_event_csv.arn}/*",
+    ]
+  }
+
+  statement {
+    sid    = "AllowKMSAccess"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:Encrypt",
+      "kms:GenerateDataKey*",
+      "kms:ReEncrypt*",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+}

infrastructure/terraform/components/app/module_queue_csv_writer_lambda.tf

@@ -0,0 +1,13 @@
+module "s3bucket_event_csv" {
+  source        = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-s3bucket.zip"
+
+  name          = "event-csv" # agent: rationale separate bucket for CSV artifacts to isolate access pattern from template internal/quarantine buckets
+
+  aws_account_id = var.aws_account_id
+  region          = var.region
+  project         = var.project
+  environment     = var.environment
+  component       = var.component
+
+  kms_key_arn     = module.kms.key_arn
+}

infrastructure/terraform/components/app/module_s3bucket_event_csv.tf

@@ -0,0 +1,13 @@
+module "sqs_event_csv" {
+  source         = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.20/terraform-sqs.zip"
+
+  aws_account_id = var.aws_account_id
+  component      = var.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  name           = "event-csv"
+  fifo_queue     = true      # agent: rationale ordering guarantees deterministic header aggregation & easier replay
+  sqs_kms_key_arn= module.kms.key_arn
+  create_dlq     = true      # agent: rationale poison message isolation for malformed JSON
+}

infrastructure/terraform/components/app/module_sqs_event_csv.tf

@@ -0,0 +1,29 @@
+# queue-csv-writer Lambda
+
+Consumes messages from the `event-csv` SQS queue and writes the `data` object from each message body to a CSV file in the `event-csv` S3 bucket.
+
+## Behaviour
+- Expects each SQS message body to be JSON containing a top-level `data` object.
+- Builds a header row from the union of all keys across received `data` objects (sorted alphabetically).
+- Escapes values per RFC4180 rules (quotes, commas, new lines).
+- Uploads the CSV to `s3://$EVENT_CSV_BUCKET_NAME/events/<ISO_TIMESTAMP>.csv`.
+- Returns `{ status: 'ok' }` or `{ status: 'no-data' }` when no rows were produced.
+
+## Environment Variables
+- `EVENT_CSV_BUCKET_NAME` – bucket to upload CSV files (injected by Terraform).
+
+## Build
+
+```bash
+npm run lambda-build
+```
+
+## Test
+
+```bash
+npm run test:unit
+```
+
+## Notes
+- IAM policy grants minimal SQS consume & S3 PutObject permissions.
+- DLQ is created by the SQS module for poison messages.

lambdas/queue-csv-writer/README.md

@@ -0,0 +1,130 @@
+import { _test, handler, createHandler } from '../src/queue-csv-writer';
+import { S3Client } from '@aws-sdk/client-s3';
+
+// Minimal mock by monkey patching send
+jest.mock('@aws-sdk/client-s3', () => {
+  const actual = jest.requireActual('@aws-sdk/client-s3');
+  return {
+    ...actual,
+    S3Client: jest.fn().mockImplementation(() => ({ send: jest.fn().mockResolvedValue({}) }))
+  };
+});
+
+describe('queue-csv-writer lambda', () => {
+  beforeEach(() => {
+    process.env.EVENT_CSV_BUCKET_NAME = 'test-bucket';
+  });
+
+  test('buildCsv produces header union and rows', () => {
+    const csv = _test.buildCsv([
+      { a: 1, b: 'x' },
+      { b: 'y', c: true }
+    ]);
+    expect(csv.split('\n')[0]).toBe('a,b,c');
+    expect(csv).toContain('1,x,'); // first row
+    expect(csv).toContain(',y,true'); // second row
+  });
+
+  test('buildCsv returns empty string for no rows', () => {
+    expect(_test.buildCsv([])).toBe('');
+  });
+
+  // Custom CSV row splitter that respects quoted multiline fields
+  function splitCsvRows(csv: string): string[] {
+    const rows: string[] = [];
+    let current = '';
+    let inQuotes = false;
+    for (let i = 0; i < csv.length; i++) {
+      const ch = csv[i];
+      if (ch === '"') {
+        // Handle escaped quotes inside a quoted field
+        if (inQuotes && csv[i + 1] === '"') {
+          current += '""';
+          i++;
+          continue;
+        }
+        inQuotes = !inQuotes;
+        current += ch;
+        continue;
+      }
+      if (ch === '\n' && !inQuotes) {
+        rows.push(current);
+        current = '';
+        continue;
+      }
+      current += ch;
+    }
+    rows.push(current);
+    return rows;
+  }
+
+  test('escapeCsv quotes commas, quotes and newlines', () => {
+    const csv = _test.buildCsv([
+      { text: 'hello, world' },
+      { text: 'line1\nline2' },
+      { text: 'He said "Hi"' }
+    ]);
+    const rows = splitCsvRows(csv);
+    expect(rows[1]).toBe('"hello, world"');
+    expect(rows[2]).toBe('"line1\nline2"');
+    expect(rows[3]).toBe('"He said ""Hi"""');
+  });
+
+  test('escapeCsv leaves simple values unquoted and null -> empty', () => {
+    const csv = _test.buildCsv([
+      { a: 'simple', b: null },
+    ]);
+    const rows = splitCsvRows(csv);
+    expect(rows[0]).toBe('a,b');
+    expect(rows[1]).toBe('simple,');
+  });
+
+  test('escapeCsv handles object value by JSON stringifying', () => {
+    const csv = _test.buildCsv([
+      { obj: { nested: 'v', n: 1 } },
+    ]);
+    const rows = splitCsvRows(csv);
+    // Object includes braces and quotes so must be quoted and escaped
+    expect(rows[0]).toBe('obj');
+    // Expect doubled quotes inside the quoted JSON per RFC4180 escaping logic
+    expect(rows[1]).toBe('"{""nested"":""v"",""n"":1}"');
+  });
+
+  test('handler uploads csv when data present', async () => {
+    const event = {
+      Records: [
+        { body: JSON.stringify({ data: { a: 1, b: 'x' } }) },
+        { body: JSON.stringify({ data: { a: 2, c: 'z' } }) }
+      ]
+    } as any;
+    // Use a fresh handler instance to avoid any prior side effects
+    const localHandler = createHandler({ s3Client: new S3Client({}) } as any);
+    const result = await localHandler(event);
+    expect(result.status).toBe('ok');
+    expect(result.rows).toBe(2);
+    expect(result.skipped).toBe(0);
+  });
+
+  test('handler throws when bucket env missing', async () => {
+    delete process.env.EVENT_CSV_BUCKET_NAME;
+    const event = { Records: [{ body: JSON.stringify({ data: { a: 1 } }) }] } as any;
+    const localHandler = createHandler({ s3Client: new S3Client({}) } as any);
+    await expect(localHandler(event)).rejects.toThrow('EVENT_CSV_BUCKET_NAME not set');
+  });
+
+  test('handler returns no-data when none present', async () => {
+    const event = { Records: [{ body: '{}' }] } as any;
+    const result = await handler(event);
+    expect(result.status).toBe('no-data');
+    expect(result.skipped).toBeGreaterThanOrEqual(1);
+  });
+
+  test('handler skips malformed JSON', async () => {
+    const event = { Records: [{ body: 'not-json' }] } as any;
+    const container = { s3Client: new S3Client({ region: 'eu-west-2' }) } as any;
+    const localHandler = createHandler(container);
+    const res = await localHandler(event);
+    expect(res.status).toBe('no-data');
+    expect(res.skipped).toBe(1);
+  });
+});

lambdas/queue-csv-writer/__tests__/handler.test.ts

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euo pipefail
+
+rm -rf dist
+
+npx esbuild \
+  --bundle \
+  --minify \
+  --sourcemap \
+  --target=es2020 \
+  --platform=node \
+  --loader:.node=file \
+  --entry-names=[name] \
+  --outdir=dist \
+  src/queue-csv-writer.ts

lambdas/queue-csv-writer/build.sh

@@ -0,0 +1,9 @@
+import type { Config } from 'jest';
+import { baseJestConfig } from 'nhs-notify-web-template-management-utils';
+
+const config: Config = {
+  ...baseJestConfig,
+  testEnvironment: 'node'
+};
+
+export default config;

lambdas/queue-csv-writer/jest.config.ts

@@ -0,0 +1,30 @@
+{
+  "name": "nhs-notify-queue-csv-writer",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "lambda-build": "./build.sh",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "test:unit": "jest",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.911.0",
+    "zod": "^4.0.17"
+  },
+  "devDependencies": {
+    "@swc/core": "^1.11.13",
+    "@swc/jest": "^0.2.37",
+    "@tsconfig/node20": "^20.1.5",
+    "@types/aws-lambda": "^8.10.148",
+    "@types/node": "^20.11.30",
+    "@types/jest": "^29.5.14",
+    "esbuild": "^0.25.9",
+    "jest": "^29.7.0",
+    "jest-mock-extended": "^3.0.7",
+    "nhs-notify-web-template-management-test-helper-utils": "^0.0.1",
+    "nhs-notify-web-template-management-utils": "^0.0.1",
+    "typescript": "^5.8.2"
+  }
+}

lambdas/queue-csv-writer/package.json

@@ -0,0 +1,8 @@
+import { S3Client } from '@aws-sdk/client-s3';
+
+export const createContainer = () => {
+  const s3Client = new S3Client({});
+  return { s3Client };
+};
+
+export type Container = ReturnType<typeof createContainer>;