Skip to content

Commit b1f44e5

Browse files
chris-elliott-nhsdchris-elliott-nhsd
committed
CCM-8861: Add SFTP poll
1 parent f289868 commit b1f44e5

22 files changed

+815
-19
lines changed

‎infrastructure/terraform/modules/backend-api/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ No requirements.
4444
| <a name="module_lambda_enrich_guardduty_scan_result"></a> [lambda\_enrich\_guardduty\_scan\_result](#module\_lambda\_enrich\_guardduty\_scan\_result) | ../lambda-function | n/a |
4545
| <a name="module_lambda_send_letter_proof"></a> [lambda\_send\_letter\_proof](#module\_lambda\_send\_letter\_proof) | ../lambda-function | n/a |
4646
| <a name="module_lambda_set_file_virus_scan_status"></a> [lambda\_set\_file\_virus\_scan\_status](#module\_lambda\_set\_file\_virus\_scan\_status) | ../lambda-function | n/a |
47+
| <a name="module_lambda_sftp_poll"></a> [lambda\_sftp\_poll](#module\_lambda\_sftp\_poll) | ../lambda-function | n/a |
4748
| <a name="module_list_template_lambda"></a> [list\_template\_lambda](#module\_list\_template\_lambda) | ../lambda-function | n/a |
4849
| <a name="module_s3bucket_internal"></a> [s3bucket\_internal](#module\_s3bucket\_internal) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
4950
| <a name="module_s3bucket_quarantine"></a> [s3bucket\_quarantine](#module\_s3bucket\_quarantine) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |

‎infrastructure/terraform/modules/backend-api/api_gateway_rest_api_main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ resource "aws_api_gateway_rest_api" "main" {
44
description = "Templates API"
55
disable_execute_api_endpoint = false
66

7-
binary_media_types = [ "multipart/form-data" ]
7+
binary_media_types = ["multipart/form-data"]
88
}

‎infrastructure/terraform/modules/backend-api/cloudwatch_event_rule_sftp_poll.tf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
resource "aws_cloudwatch_event_rule" "sftp_poll" {
2+
name = "${local.csi}-sftp-poll"
3+
schedule_expression = "rate(1 hour)" # Runs at the top of every hour
4+
}
5+
6+
resource "aws_cloudwatch_event_target" "sftp_poll" {
7+
rule = aws_cloudwatch_event_rule.sftp_poll.name
8+
arn = module.lambda_sftp_poll.function_arn
9+
}
10+
11+
resource "aws_lambda_permission" "allow_cloudwatch" {
12+
statement_id = "AllowExecutionFromCloudWatch"
13+
action = "lambda:InvokeFunction"
14+
function_name = module.lambda_sftp_poll.function_name
15+
principal = "events.amazonaws.com"
16+
source_arn = aws_cloudwatch_event_rule.sftp_poll.arn
17+
}

‎infrastructure/terraform/modules/backend-api/guardduty_malware_protection_plan_quarantine.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ resource "aws_guardduty_malware_protection_plan" "quarantine" {
44
protected_resource {
55
s3_bucket {
66
bucket_name = module.s3bucket_quarantine.id
7-
object_prefixes = ["pdf-template/", "test-data/"]
7+
object_prefixes = ["pdf-template/", "test-data/", "proofs/"]
88
}
99
}
1010

‎infrastructure/terraform/modules/backend-api/module_build_sftp_letters_lambdas.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ module "build_sftp_letters_lambdas" {
55

66
entrypoints = [
77
"src/send-proof.ts",
8+
"src/sftp-poll.ts",
89
]
910
}

‎infrastructure/terraform/modules/backend-api/module_lambda_send_letter_proof.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ module "lambda_send_letter_proof" {
1818
CREDENTIALS_TTL_SECONDS = 900
1919
CSI = local.csi
2020
ENVIRONMENT = var.environment
21+
QUARANTINE_BUCKET_NAME = module.s3bucket_quarantine.id
2122
INTERNAL_BUCKET_NAME = module.s3bucket_internal.id
2223
NODE_OPTIONS = "--enable-source-maps",
2324
REGION = var.region

‎infrastructure/terraform/modules/backend-api/module_lambda_sftp_poll.tf‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
module "lambda_sftp_poll" {
2+
source = "../lambda-function"
3+
description = "Lambda to poll the SFTP suppliers and "
4+
5+
function_name = "${local.csi}-sftp-poll"
6+
filename = module.build_sftp_letters_lambdas.zips["src/sftp-poll.ts"].path
7+
source_code_hash = module.build_sftp_letters_lambdas.zips["src/sftp-poll.ts"].base64sha256
8+
handler = "sftp-poll.handler"
9+
10+
log_retention_in_days = var.log_retention_in_days
11+
12+
execution_role_policy_document = data.aws_iam_policy_document.sftp_poll.json
13+
14+
environment_variables = {
15+
CREDENTIALS_TTL_MS = 900 * 1000
16+
CSI = local.csi
17+
DEFAULT_LETTER_SUPPLIER = local.default_letter_supplier.name
18+
ENVIRONMENT = var.environment
19+
QUARANTINE_BUCKET_NAME = module.s3bucket_quarantine.id
20+
INTERNAL_BUCKET_NAME = module.s3bucket_internal.id
21+
NODE_OPTIONS = "--enable-source-maps",
22+
REGION = var.region
23+
SEND_LOCK_TTL_MS = 50 * 1000 // visibility timeout 60s
24+
SFTP_ENVIRONMENT = local.sftp_environment
25+
TEMPLATES_TABLE_NAME = aws_dynamodb_table.templates.name
26+
}
27+
28+
timeout = 20
29+
}
30+
31+
data "aws_iam_policy_document" "sftp_poll" {
32+
statement {
33+
sid = "AllowDynamoAccess"
34+
effect = "Allow"
35+
36+
actions = [
37+
"dynamodb:UpdateItem",
38+
]
39+
40+
resources = [
41+
aws_dynamodb_table.templates.arn,
42+
]
43+
}
44+
45+
statement {
46+
sid = "AllowS3"
47+
effect = "Allow"
48+
49+
actions = [
50+
"s3:PutObject",
51+
"s3:ListBucket",
52+
]
53+
54+
resources = [module.s3bucket_quarantine.arn, "${module.s3bucket_quarantine.arn}/*"]
55+
}
56+
57+
statement {
58+
sid = "AllowSSMParameterRead"
59+
effect = "Allow"
60+
actions = [
61+
"ssm:GetParameter",
62+
"ssm:GetParametersByPath",
63+
]
64+
resources = [
65+
"arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${local.csi}/sftp-config/*",
66+
"arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${local.csi}/sftp-config",
67+
]
68+
}
69+
70+
statement {
71+
sid = "AllowKMSDynamoAccess"
72+
effect = "Allow"
73+
74+
actions = [
75+
"kms:Decrypt",
76+
"kms:DescribeKey",
77+
"kms:Encrypt",
78+
"kms:GenerateDataKey*",
79+
"kms:ReEncrypt*",
80+
]
81+
82+
resources = [
83+
var.kms_key_arn
84+
]
85+
}
86+
}

‎infrastructure/terraform/modules/lambda-function/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ No modules.
3131
| Name | Description |
3232
|------|-------------|
3333
| <a name="output_function_arn"></a> [function\_arn](#output\_function\_arn) | n/a |
34+
| <a name="output_function_name"></a> [function\_name](#output\_function\_name) | n/a |
3435
<!-- vale on -->
3536
<!-- markdownlint-enable -->
3637
<!-- END_TF_DOCS -->

‎infrastructure/terraform/modules/lambda-function/outputs.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
output "function_arn" {
22
value = aws_lambda_function.main.arn
33
}
4+
output "function_name" {
5+
value = aws_lambda_function.main.function_name
6+
}

‎lambdas/sftp-letters/src/__tests__/app/poll.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import type { FileInfo } from 'ssh2-sftp-client';
2+
import { mockDeep } from 'jest-mock-extended';
3+
import { App } from '../../app/poll';
4+
import { SftpSupplierClientRepository } from '../../infra/sftp-supplier-client-repository';
5+
import { SftpClient } from '../../infra/sftp-client';
6+
import { Logger } from 'nhs-notify-web-template-management-utils/logger';
7+
import { S3Repository } from 'nhs-notify-web-template-management-utils';
8+
9+
const mockPdfBuffer = Buffer.from([0x25, 0x50, 0x44, 0x46]);
10+
const mockMalformedPdfBuffer = Buffer.from([]);
11+
const downloadError = new Error('Download error');
12+
13+
const directoryType: FileInfo['type'] = 'd';
14+
const fileType: FileInfo['type'] = '-';
15+
16+
test('polls SFTP clients', async () => {
17+
const sftpClient = mockDeep<SftpClient>({
18+
exists: async (path: string) => {
19+
const existsMappings: Record<string, FileInfo['type']> = {
20+
'download-dir/sftp-environment/proofs': directoryType,
21+
'download-dir/sftp-environment/proofs/template-1-folder': directoryType,
22+
};
23+
24+
return existsMappings[path] ?? false;
25+
},
26+
get: async (path: string) => {
27+
if (
28+
path ===
29+
'download-dir/sftp-environment/proofs/template-1-folder/download-error.pdf'
30+
) {
31+
throw downloadError;
32+
}
33+
34+
const buffer = {
35+
'download-dir/sftp-environment/proofs/template-3.pdf': mockPdfBuffer,
36+
'download-dir/sftp-environment/proofs/template-1-folder/template-1.pdf':
37+
mockPdfBuffer,
38+
'download-dir/sftp-environment/proofs/template-1-folder/template-2.pdf':
39+
mockPdfBuffer,
40+
'download-dir/sftp-environment/proofs/template-1-folder/invalid-file.pdf':
41+
mockMalformedPdfBuffer,
42+
}[path];
43+
44+
if (!buffer) {
45+
throw new Error('File not found');
46+
}
47+
48+
return buffer;
49+
},
50+
list: async (path: string) =>
51+
({
52+
'download-dir/sftp-environment/proofs': [
53+
{
54+
name: 'template-1-folder',
55+
type: directoryType,
56+
modifyTime: Date.now(),
57+
},
58+
{
59+
name: 'template-3.pdf',
60+
type: fileType,
61+
modifyTime: Date.now(),
62+
},
63+
{
64+
name: 'folder-does-not-exist',
65+
type: directoryType,
66+
modifyTime: Date.now(),
67+
},
68+
],
69+
'download-dir/sftp-environment/proofs/template-1-folder': [
70+
{
71+
name: 'template-1.pdf',
72+
type: fileType,
73+
modifyTime: Date.now(),
74+
},
75+
{
76+
name: 'template-2.pdf',
77+
type: fileType,
78+
modifyTime: Date.now(),
79+
},
80+
{
81+
name: 'invalid-file.pdf',
82+
type: fileType,
83+
modifyTime: Date.now(),
84+
},
85+
{
86+
name: 'download-error.pdf',
87+
type: fileType,
88+
modifyTime: Date.now(),
89+
},
90+
],
91+
})[path] ?? [],
92+
});
93+
94+
const sftpSupplierClientRepository = mockDeep<SftpSupplierClientRepository>({
95+
listClients: async () => [
96+
{
97+
sftpClient,
98+
baseUploadDir: 'upload-dir',
99+
baseDownloadDir: 'download-dir',
100+
name: 'supplier',
101+
},
102+
],
103+
});
104+
105+
const mockLogger = mockDeep<Logger>();
106+
const s3Repository = mockDeep<S3Repository>();
107+
108+
const app = new App(
109+
sftpSupplierClientRepository,
110+
mockLogger,
111+
s3Repository,
112+
'sftp-environment'
113+
);
114+
115+
await app.poll();
116+
117+
expect(s3Repository.putRawData).toHaveBeenCalledTimes(3);
118+
expect(s3Repository.putRawData).toHaveBeenCalledWith(
119+
mockPdfBuffer,
120+
'proofs/template-1-folder/template-1.pdf'
121+
);
122+
expect(s3Repository.putRawData).toHaveBeenCalledWith(
123+
mockPdfBuffer,
124+
'proofs/template-1-folder/template-2.pdf'
125+
);
126+
expect(s3Repository.putRawData).toHaveBeenCalledWith(
127+
mockPdfBuffer,
128+
'proofs/template-3.pdf'
129+
);
130+
131+
expect(mockLogger.error).toHaveBeenCalledWith('PDF file failed validation', {
132+
copyPath:
133+
'download-dir/sftp-environment/proofs/template-1-folder/invalid-file.pdf',
134+
pastePath: 'proofs/template-1-folder/invalid-file.pdf',
135+
});
136+
expect(mockLogger.error).toHaveBeenCalledWith('Failed to copy file', {
137+
copyPath:
138+
'download-dir/sftp-environment/proofs/template-1-folder/download-error.pdf',
139+
pastePath: 'proofs/template-1-folder/download-error.pdf',
140+
error: downloadError,
141+
});
142+
143+
expect(sftpClient.connect).toHaveBeenCalledTimes(1);
144+
expect(sftpClient.end).toHaveBeenCalledTimes(1);
145+
});

0 commit comments

Comments
 (0)