CCM-8861: Use events for each supplier

Author: chris-elliott-nhsd

infrastructure/terraform/modules/backend-api/cloudwatch_event_rule_sftp_poll.tf

@@ -1,14 +1,22 @@
 resource "aws_cloudwatch_event_rule" "sftp_poll" {
-  name                = "${local.csi}-sftp-poll"
+  for_each = { for k, v in var.letter_suppliers : k => v if v.polling_enabled }
+
+  name                = "${local.csi}-sftp-poll-${lower(each.key)}"
   schedule_expression = "rate(1 hour)" # Runs at the top of every hour
 }
 
 resource "aws_cloudwatch_event_target" "sftp_poll" {
+  for_each = { for k, v in var.letter_suppliers : k => v if v.polling_enabled }
   rule = aws_cloudwatch_event_rule.sftp_poll.name
   arn  = module.lambda_sftp_poll.function_arn
+
+  input = {
+    supplier: each.key
+  }
 }
 
 resource "aws_lambda_permission" "allow_cloudwatch" {
+  for_each = { for k, v in var.letter_suppliers : k => v if v.polling_enabled }
   statement_id  = "AllowExecutionFromCloudWatch"
   action        = "lambda:InvokeFunction"
   function_name = module.lambda_sftp_poll.function_name

infrastructure/terraform/modules/backend-api/module_lambda_send_letter_proof.tf

@@ -17,8 +17,6 @@ module "lambda_send_letter_proof" {
   environment_variables = {
     CREDENTIALS_TTL_SECONDS = 900
     CSI                     = local.csi
-    ENVIRONMENT             = var.environment
-    QUARANTINE_BUCKET_NAME  = module.s3bucket_quarantine.id
     INTERNAL_BUCKET_NAME    = module.s3bucket_internal.id
     NODE_OPTIONS            = "--enable-source-maps",
     REGION                  = var.region

infrastructure/terraform/modules/backend-api/module_lambda_sftp_poll.tf

@@ -14,61 +14,41 @@ module "lambda_sftp_poll" {
   environment_variables = {
     CREDENTIALS_TTL_MS      = 900 * 1000
     CSI                     = local.csi
-    DEFAULT_LETTER_SUPPLIER = local.default_letter_supplier.name
-    ENVIRONMENT             = var.environment
     QUARANTINE_BUCKET_NAME  = module.s3bucket_quarantine.id
-    INTERNAL_BUCKET_NAME    = module.s3bucket_internal.id
     NODE_OPTIONS            = "--enable-source-maps",
     REGION                  = var.region
-    SEND_LOCK_TTL_MS        = 50 * 1000 // visibility timeout 60s
     SFTP_ENVIRONMENT        = local.sftp_environment
-    TEMPLATES_TABLE_NAME    = aws_dynamodb_table.templates.name
   }
 
   timeout = 20
 }
 
 data "aws_iam_policy_document" "sftp_poll" {
-  statement {
-    sid    = "AllowDynamoAccess"
-    effect = "Allow"
-
-    actions = [
-      "dynamodb:UpdateItem",
-    ]
-
-    resources = [
-      aws_dynamodb_table.templates.arn,
-    ]
-  }
 
   statement {
     sid    = "AllowS3"
     effect = "Allow"
 
     actions = [
       "s3:PutObject",
-      "s3:ListBucket",
     ]
 
-    resources = [module.s3bucket_quarantine.arn, "${module.s3bucket_quarantine.arn}/*"]
+    resources = ["${module.s3bucket_quarantine.arn}/*"]
   }
 
   statement {
     sid    = "AllowSSMParameterRead"
     effect = "Allow"
     actions = [
       "ssm:GetParameter",
-      "ssm:GetParametersByPath",
     ]
     resources = [
       "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${local.csi}/sftp-config/*",
-      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${local.csi}/sftp-config",
     ]
   }
 
   statement {
-    sid    = "AllowKMSDynamoAccess"
+    sid    = "AllowKMSAccess"
     effect = "Allow"
 
     actions = [

lambdas/sftp-letters/src/__tests__/app/poll.test.ts

@@ -92,14 +92,12 @@ test('polls SFTP clients', async () => {
   });
 
   const sftpSupplierClientRepository = mockDeep<SftpSupplierClientRepository>({
-    listClients: async () => [
-      {
-        sftpClient,
-        baseUploadDir: 'upload-dir',
-        baseDownloadDir: 'download-dir',
-        name: 'supplier',
-      },
-    ],
+    getClient: async () => ({
+      sftpClient,
+      baseUploadDir: 'upload-dir',
+      baseDownloadDir: 'download-dir',
+      name: 'supplier',
+    }),
   });
 
   const mockLogger = mockDeep<Logger>();
@@ -112,7 +110,7 @@ test('polls SFTP clients', async () => {
     'sftp-environment'
   );
 
-  await app.poll();
+  await app.poll('supplier');
 
   expect(s3Repository.putRawData).toHaveBeenCalledTimes(3);
   expect(s3Repository.putRawData).toHaveBeenCalledWith(

lambdas/sftp-letters/src/__tests__/infra/sftp-supplier-client-repository.test.ts

@@ -1,3 +1,7 @@
+import { GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm';
+import { createMockLogger } from 'nhs-notify-web-template-management-test-helper-utils/mock-logger';
+import { ICache } from 'nhs-notify-web-template-management-utils';
+import { mock } from 'jest-mock-extended';
 import 'aws-sdk-client-mock-jest';
 import { mockClient } from 'aws-sdk-client-mock';
 import { ZodError } from 'zod';
@@ -36,113 +40,6 @@ function setup() {
   return { mocks };
 }
 
-describe('listClients', () => {
-  test('returns client list', async () => {
-    const {
-      mocks: { logger, environment, ssmClient, cache },
-    } = setup();
-
-    ssmClient.on(GetParametersByPathCommand).resolvesOnce({
-      Parameters: [
-        {
-          Name: 'supplier-1',
-          Value: JSON.stringify({
-            host: 'testHost',
-            username: 'testUser',
-            privateKey: 'testKey',
-            hostKey: 'hostKey',
-            baseUploadDir: 'upload/dir',
-            baseDownloadDir: 'download/dir',
-          }),
-        },
-        {
-          Name: 'supplier-2',
-          Value: JSON.stringify({
-            host: 'testHost',
-            username: 'testUser',
-            privateKey: 'testKey',
-            hostKey: 'hostKey',
-            baseUploadDir: 'upload/dir',
-            baseDownloadDir: 'download/dir',
-          }),
-        },
-      ],
-    });
-
-    const sftpClientRepository = new SftpSupplierClientRepository(
-      environment,
-      ssmClient as unknown as SSMClient,
-      cache,
-      logger
-    );
-
-    const clients = await sftpClientRepository.listClients();
-
-    const mockSftpClients = jest.mocked(SftpClient).mock.instances;
-
-    expect(clients).toEqual([
-      {
-        sftpClient: mockSftpClients[0],
-        baseUploadDir: 'upload/dir',
-        baseDownloadDir: 'download/dir',
-        name: 'supplier-1',
-      },
-      {
-        sftpClient: mockSftpClients[1],
-        baseUploadDir: 'upload/dir',
-        baseDownloadDir: 'download/dir',
-        name: 'supplier-2',
-      },
-    ]);
-  });
-
-  test('returns empty array if no parameters are found', async () => {
-    const {
-      mocks: { logger, environment, ssmClient, cache },
-    } = setup();
-
-    ssmClient.on(GetParametersByPathCommand).resolvesOnce({
-      Parameters: undefined,
-    });
-
-    const sftpClientRepository = new SftpSupplierClientRepository(
-      environment,
-      ssmClient as unknown as SSMClient,
-      cache,
-      logger
-    );
-
-    const clients = await sftpClientRepository.listClients();
-
-    expect(clients).toEqual([]);
-  });
-
-  test('throws error if malformed SSM response is found', async () => {
-    const {
-      mocks: { logger, environment, ssmClient, cache },
-    } = setup();
-
-    ssmClient.on(GetParametersByPathCommand).resolvesOnce({
-      Parameters: [
-        {
-          Name: 'name',
-        },
-      ],
-    });
-
-    const sftpClientRepository = new SftpSupplierClientRepository(
-      environment,
-      ssmClient as unknown as SSMClient,
-      cache,
-      logger
-    );
-
-    await expect(() => sftpClientRepository.listClients()).rejects.toThrow(
-      'SFTP credentials for name are undefined'
-    );
-  });
-});
-
 describe('getClient', () => {
   test('Returns client when credentials are not cached', async () => {
     const {

lambdas/sftp-letters/src/api/sftp-poll.ts

@@ -1,6 +1,12 @@
 import type { Container } from '../container-poll';
 
+type EventBridgeEvent = {
+  event: {
+    supplier: string;
+  };
+};
+
 export const createHandler =
   ({ app }: Container) =>
-  () =>
-    app.poll();
+  ({ event: { supplier } }: EventBridgeEvent) =>
+    app.poll(supplier);

lambdas/sftp-letters/src/app/poll.ts

@@ -91,26 +91,25 @@ export class App {
     }
   }
 
-  async poll() {
-    const sftpConfigs = await this.sftpSupplierClientRepository.listClients();
+  async poll(supplier: string) {
+    const { sftpClient, baseDownloadDir } =
+      await this.sftpSupplierClientRepository.getClient(supplier);
 
-    for (const { sftpClient, baseDownloadDir, name } of sftpConfigs) {
-      this.logger.info('Polling SFTP');
-      await sftpClient.connect();
+    this.logger.info('Polling SFTP');
+    await sftpClient.connect();
 
-      this.logger.info(
-        `Copying files from folder ${baseDownloadDir} to proofs/${name}`
-      );
+    this.logger.info(
+      `Copying files from folder ${baseDownloadDir}/${this.sftpEnvironment}/proofs to proofs`
+    );
 
-      await this.copyFolder(
-        sftpClient,
-        `${baseDownloadDir}/${this.sftpEnvironment}/proofs`,
-        'proofs'
-      );
+    await this.copyFolder(
+      sftpClient,
+      `${baseDownloadDir}/${this.sftpEnvironment}/proofs`,
+      'proofs'
+    );
 
-      await sftpClient.end();
+    await sftpClient.end();
 
-      this.logger.info('Finished polling SFTP');
-    }
+    this.logger.info('Finished polling SFTP');
   }
 }

lambdas/sftp-letters/src/config/config-poll.ts

@@ -0,0 +1,22 @@
+import z from 'zod';
+
+export function loadConfig() {
+  return z
+    .object({
+      CREDENTIALS_TTL_MS: z.string().pipe(z.coerce.number()),
+      CSI: z.string(),
+      QUARANTINE_BUCKET_NAME: z.string(),
+      REGION: z.string(),
+      SFTP_ENVIRONMENT: z.string(),
+    })
+    .transform((e) => ({
+      credentialsTtlMs: e.CREDENTIALS_TTL_MS,
+      csi: e.CSI,
+      quarantineBucketName: e.QUARANTINE_BUCKET_NAME,
+      region: e.REGION,
+      sftpEnvironment: e.SFTP_ENVIRONMENT,
+    }))
+    .parse(process.env);
+}
+
+export type Config = ReturnType<typeof loadConfig>;

lambdas/sftp-letters/src/config/config.ts …s/sftp-letters/src/config/config-send.tslambdas/sftp-letters/src/config/config.ts renamed to lambdas/sftp-letters/src/config/config-send.ts lambdas/sftp-letters/src/config/config.ts renamed to lambdas/sftp-letters/src/config/config-send.ts

@@ -22,3 +22,5 @@ export function loadConfig() {
     }))
     .parse(process.env);
 }
+
+export type Config = ReturnType<typeof loadConfig>;

lambdas/sftp-letters/src/container-poll.ts

@@ -1,7 +1,7 @@
 import { S3Client } from '@aws-sdk/client-s3';
 import { SSMClient } from '@aws-sdk/client-ssm';
 import { SftpSupplierClientRepository } from './infra/sftp-supplier-client-repository';
-import { loadConfig } from './config/config';
+import { loadConfig } from './config/config-poll';
 import { App } from './app/poll';
 import { logger } from 'nhs-notify-web-template-management-utils/logger';
 import {