CCM-8574: handle test data file format

Author: harrim91

infrastructure/terraform/modules/backend-api/README.md

@@ -43,7 +43,7 @@ No requirements.
 | <a name="module_s3bucket_internal"></a> [s3bucket\_internal](#module\_s3bucket\_internal) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
 | <a name="module_s3bucket_quarantine"></a> [s3bucket\_quarantine](#module\_s3bucket\_quarantine) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
 | <a name="module_sftp_upload_queue"></a> [sftp\_upload\_queue](#module\_sftp\_upload\_queue) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v1.0.8 |
-| <a name="module_sqs_validate_letter_template_files"></a> [sqs\_validate\_letter\_template\_files](#module\_sqs\_validate\_letter\_template\_files) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v1.0.8 |
+| <a name="module_sqs_validate_letter_template_files"></a> [sqs\_validate\_letter\_template\_files](#module\_sqs\_validate\_letter\_template\_files) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.1 |
 | <a name="module_sqs_virus_scan_failed_delete_object_dlq"></a> [sqs\_virus\_scan\_failed\_delete\_object\_dlq](#module\_sqs\_virus\_scan\_failed\_delete\_object\_dlq) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v1.0.8 |
 | <a name="module_sqs_virus_scan_passed_copy_object_dlq"></a> [sqs\_virus\_scan\_passed\_copy\_object\_dlq](#module\_sqs\_virus\_scan\_passed\_copy\_object\_dlq) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v1.0.8 |
 | <a name="module_sqs_virus_scan_set_file_status_dlq"></a> [sqs\_virus\_scan\_set\_file\_status\_dlq](#module\_sqs\_virus\_scan\_set\_file\_status\_dlq) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v1.0.8 |

infrastructure/terraform/modules/backend-api/module_sqs_validate_letter_template_files.tf

@@ -1,5 +1,5 @@
 module "sqs_validate_letter_template_files" {
-  source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v1.0.8"
+  source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v2.0.1"
 
   aws_account_id             = var.aws_account_id
   component                  = var.component
@@ -9,6 +9,7 @@ module "sqs_validate_letter_template_files" {
   name                       = "validate-letter-template-files"
   create_dlq                 = true
   visibility_timeout_seconds = 10
+  delay_seconds              = 1
 
   sqs_kms_key_arn = var.kms_key_arn
 }

lambdas/backend-api/src/__tests__/templates/api/validate-letter-template-files.test.ts

@@ -90,7 +90,7 @@ describe('guard duty handler', () => {
     });
     mocks.TemplatePdf.mockImplementation(() => pdf);
 
-    const csv = mock<TestDataCsv>({ headers: ['parameter_1'] });
+    const csv = mock<TestDataCsv>({ parameters: ['parameter_1'] });
     mocks.TestDataCsv.mockImplementation(() => csv);
 
     mocks.validateLetterTemplateFiles.mockReturnValueOnce(true);
@@ -129,7 +129,7 @@ describe('guard duty handler', () => {
       versionId,
       true,
       pdf.personalisationParameters,
-      csv.headers
+      csv.parameters
     );
   });
 
@@ -958,7 +958,7 @@ describe('guard duty handler', () => {
     });
     mocks.TemplatePdf.mockImplementation(() => pdf);
 
-    const csv = mock<TestDataCsv>({ headers: ['parameter_1'] });
+    const csv = mock<TestDataCsv>({ parameters: ['parameter_1'] });
     mocks.TestDataCsv.mockImplementation(() => csv);
 
     mocks.validateLetterTemplateFiles.mockReturnValue(false);
@@ -972,7 +972,7 @@ describe('guard duty handler', () => {
       versionId,
       false,
       pdf.personalisationParameters,
-      csv.headers
+      csv.parameters
     );
   });
 });

lambdas/backend-api/src/__tests__/templates/domain/test-data-csv.test.ts

@@ -12,32 +12,32 @@ test('parse', async () => {
 
   csv.parse();
 
-  expect(csv.headers).toEqual([
+  expect(csv.parameters).toEqual([
     'appointment_date',
     'appointment_time',
     'appointment_location',
     'contact_telephone_number',
   ]);
 
-  expect(csv.rows).toEqual([
-    [
-      'Wednesday 10 September 2025',
-      '12:56pm',
-      'City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH',
-      '020 3299 9010',
-    ],
-    [
-      'Saturday 10 April 2025',
-      '11:56am',
-      'The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ',
-      '020 3299 9010',
-    ],
-    [
-      'Monday 1 May 2025',
-      '1:56pm',
-      "The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN",
-      '020 3299 9010',
-    ],
+  expect(csv.short).toEqual([
+    'Wednesday 10 September 2025',
+    '12:56pm',
+    'City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH',
+    '020 3299 9010',
+  ]);
+
+  expect(csv.medium).toEqual([
+    'Saturday 10 April 2025',
+    '11:56am',
+    'The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ',
+    '020 3299 9010',
+  ]);
+
+  expect(csv.long).toEqual([
+    'Monday 1 May 2025',
+    '1:56pm',
+    "The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN",
+    '020 3299 9010',
   ]);
 });
 
@@ -48,16 +48,22 @@ test('errors if parse is not called before reading personalisation', () => {
 
   const csv = new TestDataCsv(file);
 
-  expect(() => csv.headers).toThrow('CSV has not been parsed');
-  expect(() => csv.rows).toThrow('CSV has not been parsed');
+  expect(() => csv.parameters).toThrow('CSV has not been parsed');
+  expect(() => csv.short).toThrow('CSV has not been parsed');
+  expect(() => csv.medium).toThrow('CSV has not been parsed');
+  expect(() => csv.long).toThrow('CSV has not been parsed');
 });
 
-test('errors if file cannot be parsed', async () => {
-  const file = fs.readFileSync(
-    path.resolve(__dirname, '../fixtures/custom-personalisation.pdf')
-  );
+test.each([
+  ['not a csv', '../fixtures/custom-personalisation.pdf'],
+  ['wrong file headers', '../fixtures/wrong-headers.csv'],
+  ['badly ordered file headers', '../fixtures/wrong-header-order.csv'],
+  ['missing example', '../fixtures/missing-example.csv'],
+  ['missing parameter in example', '../fixtures/missing-parameter.csv'],
+])('errors if file cannot be parsed - %s', async (_title, filepath) => {
+  const file = fs.readFileSync(path.resolve(__dirname, filepath));
 
   const csv = new TestDataCsv(file);
 
-  expect(csv.parse).toThrow();
+  expect(() => csv.parse()).toThrow();
 });

lambdas/backend-api/src/__tests__/templates/domain/validate-letter-template-files.test.ts

@@ -68,7 +68,7 @@ describe('pdf has no custom personalisation', () => {
       ],
     });
 
-    const csv = mock<TestDataCsv>({ headers: [], rows: [] });
+    const csv = mock<TestDataCsv>({ parameters: [] });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(true);
   });
@@ -88,8 +88,7 @@ describe('pdf has no custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['parameter_1'],
-      rows: [['value_1']],
+      parameters: ['parameter_1'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -135,8 +134,7 @@ describe('pdf with custom personalisation', () => {
       });
 
       const csv = mock<TestDataCsv>({
-        headers: [value],
-        rows: [['value_1']],
+        parameters: [value],
       });
 
       expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -158,8 +156,7 @@ describe('pdf with custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['parameter_2'],
-      rows: [['value_2']],
+      parameters: ['parameter_2'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -180,8 +177,7 @@ describe('pdf with custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['parameter_1', 'parameter_2'],
-      rows: [['value_1', 'value_2']],
+      parameters: ['parameter_1', 'parameter_2'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -202,8 +198,7 @@ describe('pdf with custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['parameter_2'],
-      rows: [['value_2']],
+      parameters: ['parameter_2'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -224,52 +219,7 @@ describe('pdf with custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['fullName'],
-      rows: [['example name']],
-    });
-
-    expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
-  });
-
-  test('returns false if csv row contains too few values', () => {
-    const pdf = mock<TemplatePdf>({
-      customPersonalisationParameters: ['parameter_1'],
-      addressLinePersonalisationParameters: [
-        'address_line_1',
-        'address_line_2',
-        'address_line_3',
-        'address_line_4',
-        'address_line_5',
-        'address_line_6',
-        'address_line_7',
-      ],
-    });
-
-    const csv = mock<TestDataCsv>({
-      headers: ['parameter_1'],
-      rows: [['value_1'], []],
-    });
-
-    expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
-  });
-
-  test('returns false if csv row contains too many values', () => {
-    const pdf = mock<TemplatePdf>({
-      customPersonalisationParameters: ['parameter_1'],
-      addressLinePersonalisationParameters: [
-        'address_line_1',
-        'address_line_2',
-        'address_line_3',
-        'address_line_4',
-        'address_line_5',
-        'address_line_6',
-        'address_line_7',
-      ],
-    });
-
-    const csv = mock<TestDataCsv>({
-      headers: ['parameter_1'],
-      rows: [['value_1'], ['value_2', 'value_3']],
+      parameters: ['fullName'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(false);
@@ -290,11 +240,7 @@ describe('pdf with custom personalisation', () => {
     });
 
     const csv = mock<TestDataCsv>({
-      headers: ['parameter_1', 'parameter_2'],
-      rows: [
-        ['value_1', 'value_2'],
-        ['value_3', 'value_4'],
-      ],
+      parameters: ['parameter_1', 'parameter_2'],
     });
 
     expect(validateLetterTemplateFiles(pdf, csv)).toBe(true);

lambdas/backend-api/src/__tests__/templates/fixtures/missing-example.csv

@@ -0,0 +1,5 @@
+Personalisation field,Short length data example,Medium length data example,Long length data example
+appointment_date,Wednesday 10 September 2025,Saturday 10 April 2025
+appointment_time,12:56pm,11:56am
+appointment_location,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH","The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ"
+contact_telephone_number,020 3299 9010,020 3299 9010

lambdas/backend-api/src/__tests__/templates/fixtures/missing-parameter.csv

@@ -0,0 +1,5 @@
+Personalisation field,Short length data example,Medium length data example,Long length data example
+appointment_date,Wednesday 10 September 2025,Saturday 10 April 2025,Monday 1 May 2025
+appointment_time,12:56pm,11:56am,1:56pm
+appointment_location,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH","The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ","The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN"
+contact_telephone_number,,020 3299 9010,020 3299 9010

lambdas/backend-api/src/__tests__/templates/fixtures/test-data.csv

@@ -1,4 +1,5 @@
-appointment_date,appointment_time,appointment_location,contact_telephone_number
-Wednesday 10 September 2025,12:56pm,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH",020 3299 9010
-Saturday 10 April 2025,11:56am,"The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ",020 3299 9010
-Monday 1 May 2025,1:56pm,"The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN",020 3299 9010
+Personalisation field,Short length data example,Medium length data example,Long length data example
+appointment_date,Wednesday 10 September 2025,Saturday 10 April 2025,Monday 1 May 2025
+appointment_time,12:56pm,11:56am,1:56pm
+appointment_location,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH","The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ","The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN"
+contact_telephone_number,020 3299 9010,020 3299 9010,020 3299 9010

lambdas/backend-api/src/__tests__/templates/fixtures/wrong-header-order.csv

@@ -0,0 +1,5 @@
+Personalisation field,Long length data example,Medium length data example,Short length data example
+appointment_date,Wednesday 10 September 2025,Saturday 10 April 2025,Monday 1 May 2025
+appointment_time,12:56pm,11:56am,1:56pm
+appointment_location,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH","The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ","The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN"
+contact_telephone_number,020 3299 9010,020 3299 9010,020 3299 9010

lambdas/backend-api/src/__tests__/templates/fixtures/wrong-headers.csv

@@ -0,0 +1,5 @@
+personalisation_field,example_1,example_2,example_3
+appointment_date,Wednesday 10 September 2025,Saturday 10 April 2025,Monday 1 May 2025
+appointment_time,12:56pm,11:56am,1:56pm
+appointment_location,"City, Sandwell & Walsall BSS, The Rosewood Centre, Sandwell & West Birmingham Hospitals NHS Trust, The Birmingham Treatment Centre, City Hospital, Dudley Road, Birmingham, B18 7QH","The Royal Shrewsbury Hospital, Breast Screening Office, Treatment Centre, Mytton Oak Road, Shrewsbury, SY3 8XQ","The Epping Breast Screening Unit, St Margaret's Hospital, The Plain, Epping, Essex, CM16 6TN"
+contact_telephone_number,020 3299 9010,020 3299 9010,020 3299 9010