CCM-8861: SFTP poll lambda

chris-elliott-nhsd · chris-elliott-nhsd · commit 8ffea30b33e9 · 2025-04-14T19:38:44.000+01:00
diff --git a/infrastructure/terraform/components/app/variables.tf b/infrastructure/terraform/components/app/variables.tf
@@ -198,6 +198,6 @@ variable "letter_suppliers" {
     enable_polling   = bool
     default_supplier = optional(bool)
   }))
-  default = {}
+  default     = {}
   description = "Letter suppliers enabled in the environment"
 }
diff --git a/infrastructure/terraform/modules/backend-api/README.md b/infrastructure/terraform/modules/backend-api/README.md
@@ -43,6 +43,7 @@ No requirements.
 | <a name="module_lambda_enrich_guardduty_scan_result"></a> [lambda\_enrich\_guardduty\_scan\_result](#module\_lambda\_enrich\_guardduty\_scan\_result) | ../lambda-function | n/a |
 | <a name="module_lambda_send_letter_proof"></a> [lambda\_send\_letter\_proof](#module\_lambda\_send\_letter\_proof) | ../lambda-function | n/a |
 | <a name="module_lambda_set_file_virus_scan_status"></a> [lambda\_set\_file\_virus\_scan\_status](#module\_lambda\_set\_file\_virus\_scan\_status) | ../lambda-function | n/a |
+| <a name="module_lambda_sftp_poll"></a> [lambda\_sftp\_poll](#module\_lambda\_sftp\_poll) | ../lambda-function | n/a |
 | <a name="module_list_template_lambda"></a> [list\_template\_lambda](#module\_list\_template\_lambda) | ../lambda-function | n/a |
 | <a name="module_s3bucket_internal"></a> [s3bucket\_internal](#module\_s3bucket\_internal) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
 | <a name="module_s3bucket_quarantine"></a> [s3bucket\_quarantine](#module\_s3bucket\_quarantine) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
diff --git a/infrastructure/terraform/modules/backend-api/api_gateway_rest_api_main.tf b/infrastructure/terraform/modules/backend-api/api_gateway_rest_api_main.tf
@@ -4,5 +4,5 @@ resource "aws_api_gateway_rest_api" "main" {
   description                  = "Templates API"
   disable_execute_api_endpoint = false
 
-  binary_media_types = [ "multipart/form-data" ]
+  binary_media_types = ["multipart/form-data"]
 }
diff --git a/infrastructure/terraform/modules/backend-api/cloudwatch_event_rule_sftp_poll.tf b/infrastructure/terraform/modules/backend-api/cloudwatch_event_rule_sftp_poll.tf
@@ -0,0 +1,17 @@
+resource "aws_cloudwatch_event_rule" "sftp_poll" {
+  name                = "${local.csi}-sftp-poll"
+  schedule_expression = "rate(1 hour)" # Runs at the top of every hour
+}
+
+resource "aws_cloudwatch_event_target" "lambda_target" {
+  rule = aws_cloudwatch_event_rule.sftp_poll.name
+  arn  = module.lambda_sftp_poll.function_name
+}
+
+resource "aws_lambda_permission" "allow_cloudwatch" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = module.lambda_sftp_poll.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.sftp_poll.arn
+}
diff --git a/infrastructure/terraform/modules/backend-api/module_build_sftp_letters_lambdas.tf b/infrastructure/terraform/modules/backend-api/module_build_sftp_letters_lambdas.tf
@@ -5,5 +5,6 @@ module "build_sftp_letters_lambdas" {
 
   entrypoints = [
     "src/send-proof.ts",
+    "src/sftp-poll.ts",
   ]
 }
diff --git a/infrastructure/terraform/modules/backend-api/module_lambda_send_letter_proof.tf b/infrastructure/terraform/modules/backend-api/module_lambda_send_letter_proof.tf
@@ -19,6 +19,7 @@ module "lambda_send_letter_proof" {
     CSI                     = local.csi
     DEFAULT_LETTER_SUPPLIER = local.default_letter_supplier.name
     ENVIRONMENT             = var.environment
+    QUARANTINE_BUCKET_NAME  = module.s3bucket_quarantine.id
     INTERNAL_BUCKET_NAME    = module.s3bucket_internal.id
     NODE_OPTIONS            = "--enable-source-maps",
     REGION                  = var.region
diff --git a/infrastructure/terraform/modules/backend-api/module_lambda_sftp_poll.tf b/infrastructure/terraform/modules/backend-api/module_lambda_sftp_poll.tf
@@ -0,0 +1,78 @@
+module "lambda_sftp_poll" {
+  source      = "../lambda-function"
+  description = "Lambda to poll the SFTP suppliers and "
+
+  function_name    = "${local.csi}-sftp-poll"
+  filename         = module.build_sftp_letters_lambdas.zips["src/sftp-poll.ts"].path
+  source_code_hash = module.build_sftp_letters_lambdas.zips["src/sftp-poll.ts"].base64sha256
+  handler          = "sftp-poll.handler"
+
+  log_retention_in_days = var.log_retention_in_days
+
+  execution_role_policy_document = data.aws_iam_policy_document.sftp_poll.json
+
+  environment_variables = {
+    CSI                     = local.csi
+    ENVIRONMENT             = var.environment
+    QUARANTINE_BUCKET_NAME  = module.s3bucket_quarantine.id
+    INTERNAL_BUCKET_NAME    = module.s3bucket_internal.id
+    DEFAULT_LETTER_SUPPLIER = local.default_letter_supplier.name
+    SFTP_ENVIRONMENT        = local.sftp_environment
+    REGION                  = var.region
+    NODE_OPTIONS            = "--enable-source-maps",
+  }
+}
+
+data "aws_iam_policy_document" "sftp_poll" {
+  statement {
+    sid    = "AllowDynamoAccess"
+    effect = "Allow"
+
+    actions = [
+      "dynamodb:UpdateItem",
+    ]
+
+    resources = [
+      aws_dynamodb_table.templates.arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+    ]
+
+    resources = ["${module.s3bucket_quarantine.arn}/*"]
+  }
+
+  statement {
+    sid    = "AllowSSMParameterRead"
+    effect = "Allow"
+    actions = [
+      "ssm:GetParameter",
+    ]
+    resources = [
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${local.csi}/sftp-config/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowKMSDynamoAccess"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:Encrypt",
+      "kms:GenerateDataKey*",
+      "kms:ReEncrypt*",
+    ]
+
+    resources = [
+      var.kms_key_arn
+    ]
+  }
+}
diff --git a/infrastructure/terraform/modules/lambda-function/README.md b/infrastructure/terraform/modules/lambda-function/README.md
@@ -32,6 +32,7 @@ No modules.
 | Name | Description |
 |------|-------------|
 | <a name="output_function_arn"></a> [function\_arn](#output\_function\_arn) | n/a |
+| <a name="output_function_name"></a> [function\_name](#output\_function\_name) | n/a |
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->
diff --git a/infrastructure/terraform/modules/lambda-function/outputs.tf b/infrastructure/terraform/modules/lambda-function/outputs.tf
@@ -1,3 +1,6 @@
 output "function_arn" {
   value = aws_lambda_function.main.arn
 }
+output "function_name" {
+  value = aws_lambda_function.main.function_name
+}
diff --git a/lambdas/sftp-letters/src/__tests__/infra/sftp-supplier-client-repository.test.ts b/lambdas/sftp-letters/src/__tests__/infra/sftp-supplier-client-repository.test.ts
@@ -70,6 +70,7 @@ describe('getClient', () => {
       sftpClient: mockSftpClient,
       baseUploadDir: 'upload/dir',
       baseDownloadDir: 'download/dir',
+      name: 'SYNERTEC',
     });
 
     expect(cache.get).toHaveBeenCalledWith(credKey);
@@ -121,6 +122,7 @@ describe('getClient', () => {
       sftpClient: mockSftpClient,
       baseUploadDir: 'upload/dir',
       baseDownloadDir: 'download/dir',
+      name: 'SYNERTEC',
     });
 
     expect(cache.get).toHaveBeenCalledWith(credKey);
diff --git a/lambdas/sftp-letters/src/api/sftp-poll.ts b/lambdas/sftp-letters/src/api/sftp-poll.ts
@@ -0,0 +1,6 @@
+import type { Container } from '../container-poll';
+
+export const createHandler =
+  ({ app }: Container) =>
+  () =>
+    app.poll();
diff --git a/lambdas/sftp-letters/src/app/poll.ts b/lambdas/sftp-letters/src/app/poll.ts
@@ -0,0 +1,115 @@
+import type { Logger } from 'nhs-notify-web-template-management-utils/logger';
+import type { SftpSupplierClientRepository } from '../infra/sftp-supplier-client-repository';
+import type { SftpClient } from '../infra/sftp-client';
+import type { S3Repository } from 'nhs-notify-web-template-management-utils';
+
+export class App {
+  constructor(
+    private readonly sftpSupplierClientRepository: SftpSupplierClientRepository,
+    private readonly logger: Logger,
+    private readonly s3Repository: S3Repository
+  ) {}
+
+  private async copyFolder(
+    sftpClient: SftpClient,
+    copyPath: string,
+    pastePath: string
+  ) {
+    const isDir = (await sftpClient.exists(copyPath)) === 'd';
+
+    if (!isDir) {
+      this.logger.info(`Path '${copyPath}' does not exist`);
+      return;
+    }
+
+    const sftpFiles = await sftpClient.list(copyPath);
+
+    const s3Files = await this.s3Repository.listObjects(
+      `${pastePath}/`,
+      '/' // this delimiter causes listS3objects to return only objects at first level of key 'directory'
+    );
+
+    const s3FileMetadata = s3Files.map((val) => {
+      const name = val.Key?.split('/').pop() || '';
+      return {
+        name,
+        modifyTime: val.LastModified?.getTime() || Date.now(),
+      };
+    });
+
+    for (const sftpFile of sftpFiles) {
+      if (this.shouldFileBeCopied(sftpFile, s3FileMetadata)) {
+        if (sftpFile.type === 'd') {
+          // recurse
+          await this.copyFolder(
+            sftpClient,
+            `${copyPath}/${sftpFile.name}`,
+            `${pastePath}/${sftpFile.name}`
+          );
+        }
+        if (sftpFile.type === '-') {
+          await this.copyFile(
+            sftpClient,
+            `${copyPath}/${sftpFile.name}`,
+            `${pastePath}/${sftpFile.name}`
+          );
+        }
+      }
+    }
+  }
+
+  private shouldFileBeCopied(
+    sftpFile: { name: string; modifyTime: number },
+    s3Files: { name: string; modifyTime: number }[]
+  ) {
+    const s3File = s3Files.find((file) => file.name === sftpFile.name);
+    return !s3File || s3File.modifyTime < sftpFile.modifyTime;
+  }
+
+  private async copyFile(
+    sftpClient: SftpClient,
+    copyPath: string,
+    pastePath: string
+  ) {
+    try {
+      // docs specify undefined destination will return Buffer type
+      // however, empty files seem to return an empty array instead of a buffer
+      const data = (await sftpClient.get(`${copyPath}`)) as Buffer | [];
+
+      const buffer = Array.isArray(data) ? Buffer.from(data) : data;
+
+      await this.s3Repository.putRawData(buffer, pastePath);
+    } catch (error) {
+      this.logger.error({
+        description: `Failed to copy file from ${copyPath} to ${pastePath}`,
+        err: error,
+      });
+    }
+  }
+
+  async poll() {
+    const sftpConfigs = await this.sftpSupplierClientRepository.listClients(
+      this.logger
+    );
+
+    for (const {
+      sftpClient,
+      baseUploadDir,
+      baseDownloadDir,
+      name,
+    } of sftpConfigs) {
+      this.logger.info('Polling SFTP');
+      await sftpClient.connect();
+
+      this.logger.info(
+        `Copying files from folder ${baseUploadDir} to ${baseDownloadDir}`
+      );
+
+      await this.copyFolder(sftpClient, baseDownloadDir, `proofs/${name}`);
+
+      await sftpClient.end();
+
+      this.logger.info('Finished polling SFTP');
+    }
+  }
+}
diff --git a/lambdas/sftp-letters/src/config/config.ts b/lambdas/sftp-letters/src/config/config.ts
@@ -6,6 +6,7 @@ export function loadConfig() {
       CREDENTIALS_TTL_MS: z.string().pipe(z.coerce.number()),
       CSI: z.string(),
       DEFAULT_LETTER_SUPPLIER: z.string(),
+      QUARANTINE_BUCKET_NAME: z.string(),
       INTERNAL_BUCKET_NAME: z.string(),
       REGION: z.string(),
       SEND_LOCK_TTL_MS: z.string().pipe(z.coerce.number()),
@@ -16,6 +17,7 @@ export function loadConfig() {
       credentialsTtlMs: e.CREDENTIALS_TTL_MS,
       csi: e.CSI,
       defaultSupplier: e.DEFAULT_LETTER_SUPPLIER,
+      quarantineBucketName: e.QUARANTINE_BUCKET_NAME,
       internalBucketName: e.INTERNAL_BUCKET_NAME,
       region: e.REGION,
       sendLockTtlMs: e.SEND_LOCK_TTL_MS,
diff --git a/lambdas/sftp-letters/src/container-poll.ts b/lambdas/sftp-letters/src/container-poll.ts
@@ -0,0 +1,37 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { SSMClient } from '@aws-sdk/client-ssm';
+import { SftpSupplierClientRepository } from './infra/sftp-supplier-client-repository';
+import { loadConfig } from './config/config';
+import { App } from './app/poll';
+import { logger } from 'nhs-notify-web-template-management-utils/logger';
+import {
+  InMemoryCache,
+  S3Repository,
+} from 'nhs-notify-web-template-management-utils';
+
+export function createContainer() {
+  const { csi, quarantineBucketName, credentialsTtlMs, region } = loadConfig();
+
+  const ssmClient = new SSMClient({ region });
+
+  const s3Client = new S3Client({ region });
+
+  const cache = new InMemoryCache({ ttl: credentialsTtlMs });
+
+  const sftpSupplierClientRepository = new SftpSupplierClientRepository(
+    csi,
+    ssmClient,
+    cache,
+    logger
+  );
+
+  const s3Repository = new S3Repository(quarantineBucketName, s3Client);
+
+  const app = new App(sftpSupplierClientRepository, logger, s3Repository);
+
+  return {
+    app,
+  };
+}
+
+export type Container = ReturnType<typeof createContainer>;
diff --git a/lambdas/sftp-letters/src/infra/sftp-supplier-client-repository.ts b/lambdas/sftp-letters/src/infra/sftp-supplier-client-repository.ts
diff --git a/lambdas/sftp-letters/src/sftp-poll.ts b/lambdas/sftp-letters/src/sftp-poll.ts
diff --git a/utils/utils/package.json b/utils/utils/package.json
diff --git a/utils/utils/src/index.ts b/utils/utils/src/index.ts
diff --git a/utils/utils/src/s3-repository.ts b/utils/utils/src/s3-repository.ts

Original file line number	Diff line number	Diff line change
`@@ -198,6 +198,6 @@ variable "letter_suppliers" {`
`198`	`198`	`enable_polling = bool`
`199`	`199`	`default_supplier = optional(bool)`
`200`	`200`	`}))`
`201`		`- default = {}`
	`201`	`+ default = {}`
`202`	`202`	`description = "Letter suppliers enabled in the environment"`
`203`	`203`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,5 @@ resource "aws_api_gateway_rest_api" "main" {`
`4`	`4`	`description = "Templates API"`
`5`	`5`	`disable_execute_api_endpoint = false`
`6`	`6`
`7`		`- binary_media_types = [ "multipart/form-data" ]`
	`7`	`+ binary_media_types = ["multipart/form-data"]`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,5 +5,6 @@ module "build_sftp_letters_lambdas" {`
`5`	`5`
`6`	`6`	`entrypoints = [`
`7`	`7`	`"src/send-proof.ts",`
	`8`	`+ "src/sftp-poll.ts",`
`8`	`9`	`]`
`9`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,6 @@`
`1`	`1`	`output "function_arn" {`
`2`	`2`	`value = aws_lambda_function.main.arn`
`3`	`3`	`}`
	`4`	`+output "function_name" {`
	`5`	`+ value = aws_lambda_function.main.function_name`
	`6`	`+}`