CCM-11532: refactor

Author: bhansell1

package-lock.json

@@ -6,7 +6,6 @@ declare global {
       COGNITO_USER_POOL_CLIENT_ID: string;
       COGNITO_USER_POOL_ID: string;
       EVENT_CACHE_BUCKET_NAME: string;
-      NEXT_PUBLIC_ENABLE_PROOFING: string;
       PLAYWRIGHT_RUN_ID: string;
       REQUEST_PROOF_QUEUE_URL: string;
       SFTP_ENVIRONMENT: string;
@@ -16,6 +15,7 @@ declare global {
       TEMPLATES_QUARANTINE_BUCKET_NAME: string;
       TEMPLATES_DOWNLOAD_BUCKET_NAME: string;
       TEST_EMAIL_BUCKET_PREFIX: string;
+      TEST_EMAIL_BUCKET_NAME: string;
     }
   }
 

tests/test-team/global.d.ts

@@ -1,9 +1,6 @@
-import {
-  S3Client,
-  ListObjectsV2Command,
-  GetObjectCommand,
-} from '@aws-sdk/client-s3';
+import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3';
 import { Readable } from 'node:stream';
+import { S3Helper } from './s3-helper';
 
 async function streamToString(stream: Readable): Promise<string> {
   return await new Promise((resolve, reject) => {
@@ -21,50 +18,15 @@ export class EmailHelper {
 
   constructor() {}
 
-  async getAllS3Items(prefix: string) {
-    const { Contents = [], ContinuationToken } = await this.s3Client.send(
-      new ListObjectsV2Command({
-        Bucket: this.testEmailBucketName,
-        Prefix: prefix,
-      })
-    );
-
-    let nextToken = ContinuationToken;
-    let s3Items = Contents;
-
-    while (nextToken) {
-      const {
-        Contents: newContents = [],
-        ContinuationToken: newContinuationToken,
-      } = await this.s3Client.send(
-        new ListObjectsV2Command({
-          Bucket: this.testEmailBucketName,
-          Prefix: prefix,
-          ContinuationToken: nextToken,
-        })
-      );
-
-      s3Items = [...s3Items, ...newContents];
-      nextToken = newContinuationToken;
-    }
-
-    return s3Items;
-  }
-
   async getEmailForTemplateId(
     prefix: string,
     templateId: string,
     dateCutoff: Date,
     extraTextToSearch: string
   ) {
-    const s3Items = await this.getAllS3Items(prefix);
+    const s3Items = await S3Helper.listAll(this.testEmailBucketName, prefix);
 
-    const sortedKeys = s3Items
-      .filter(({ LastModified }) => (LastModified ?? 0) > dateCutoff)
-      .sort(
-        (a, b) =>
-          (b.LastModified?.getTime() ?? 0) - (a.LastModified?.getTime() ?? 0)
-      );
+    const sortedKeys = S3Helper.filterAndSort(s3Items, dateCutoff);
 
     // SES does not tell us what the S3 keys are going to be for received emails,
     // so we have to search all recent ones to ensure we get the right one and

tests/test-team/helpers/email-helper.ts

@@ -1,16 +1,16 @@
 import { z } from 'zod';
 import {
-  ListObjectsV2Command,
   S3Client,
   SelectObjectContentCommand,
   SelectObjectContentEventStream,
-  _Object,
 } from '@aws-sdk/client-s3';
 import {
   $TemplateCompletedEventV1,
   $TemplateDeletedEventV1,
   $TemplateDraftedEventV1,
 } from '@nhsdigital/nhs-notify-event-schemas-template-management';
+import { differenceInSeconds, addHours } from 'date-fns';
+import { S3Helper } from '../s3-helper';
 
 const $NHSNotifyTemplateEvent = z.discriminatedUnion('type', [
   $TemplateCompletedEventV1,
@@ -32,57 +32,21 @@ export class EventCacheHelper {
       return [];
     }
 
-    const files = await this.getAllS3items(this.buildEventCachePrefix(from));
+    const files = await Promise.all(
+      this.filePaths(from).map((path) => {
+        return S3Helper.listAll(this.bucketName, path);
+      })
+    );
 
-    const filteredFiles = this.filterAndSortFiles(files, from);
+    const filteredFiles = S3Helper.filterAndSort(files.flat(), from);
 
     const eventPromises = filteredFiles.map((file) =>
       this.queryFileForEvents(file.Key!, templateIds)
     );
 
     const results = await Promise.all(eventPromises);
 
-    return results
-      .flat()
-      .sort(
-        (a, b) =>
-          new Date(a.data.updatedAt).getTime() -
-          new Date(b.data.updatedAt).getTime()
-      );
-  }
-
-  private filterAndSortFiles(files: _Object[], from: Date): _Object[] {
-    return files
-      .filter(
-        ({ LastModified }) => (LastModified?.getTime() ?? 0) > from.getTime()
-      )
-      .sort(
-        (a, b) =>
-          (b.LastModified?.getTime() ?? 0) - (a.LastModified?.getTime() ?? 0)
-      );
-  }
-
-  private async getAllS3items(prefix: string): Promise<_Object[]> {
-    const allItems: _Object[] = [];
-    let continuationToken: string | undefined;
-
-    do {
-      const command = new ListObjectsV2Command({
-        Bucket: this.bucketName,
-        Prefix: prefix,
-        ContinuationToken: continuationToken,
-      });
-
-      const response = await this.s3.send(command);
-
-      if (response.Contents) {
-        allItems.push(...response.Contents);
-      }
-
-      continuationToken = response.NextContinuationToken;
-    } while (continuationToken);
-
-    return allItems;
+    return results.flat();
   }
 
   private async queryFileForEvents(
@@ -109,10 +73,10 @@ export class EventCacheHelper {
       return [];
     }
 
-    return await this.processS3SelectResponse(fileName, response.Payload);
+    return await this.parse(fileName, response.Payload);
   }
 
-  private async processS3SelectResponse(
+  private async parse(
     fileName: string,
     payload: AsyncIterable<SelectObjectContentEventStream>
   ): Promise<NHSNotifyTemplateEvent[]> {
@@ -152,6 +116,27 @@ export class EventCacheHelper {
     return events;
   }
 
+  /*
+   * Get files paths for the current hour
+   * and next hour if the different in seconds is greater than toleranceInSeconds
+   *
+   * The way firehose stores files is yyyy/mm/dd/hh.
+   * On a boundary of 15:59:58 you'll find files in both 15 and 16 hour folders
+   */
+  private filePaths(start: Date, toleranceInSeconds = 30): string[] {
+    const paths = [this.getEventCachePrefix(start)];
+
+    const end = addHours(start, 1);
+
+    const difference = differenceInSeconds(end, start);
+
+    if (difference >= toleranceInSeconds) {
+      paths.push(this.getEventCachePrefix(end));
+    }
+
+    return paths;
+  }
+
   private buildS3Query(templateIds: string[]): string {
     const likeConditions = templateIds
       .map((id) => `s.Message LIKE '%${id}%'`)
@@ -160,7 +145,7 @@ export class EventCacheHelper {
     return `SELECT * FROM S3Object s WHERE ${likeConditions}`;
   }
 
-  private buildEventCachePrefix(date: Date): string {
+  private getEventCachePrefix(date: Date): string {
     return date
       .toISOString()
       .slice(0, 13)

tests/test-team/helpers/events/event-cache-helper.ts

@@ -0,0 +1,45 @@
+import { _Object, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
+
+export class S3Helper {
+  private static readonly client = new S3Client({ region: 'eu-west-2' });
+
+  static async listAll(bucket: string, prefix: string): Promise<_Object[]> {
+    const allItems: _Object[] = [];
+    let continuationToken: string | undefined;
+
+    do {
+      const command = new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      });
+
+      const response = await this.client.send(command);
+
+      if (response.Contents) {
+        allItems.push(...response.Contents);
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+
+    return allItems;
+  }
+
+  static filterAndSort(files: _Object[], from: Date): _Object[] {
+    return S3Helper.sort(S3Helper.filter([...files], from));
+  }
+
+  static filter(files: _Object[], from: Date): _Object[] {
+    return files.filter(
+      ({ LastModified }) => (LastModified?.getTime() ?? 0) > from.getTime()
+    );
+  }
+
+  static sort(files: _Object[]): _Object[] {
+    return files.sort(
+      (a, b) =>
+        (b.LastModified?.getTime() ?? 0) - (a.LastModified?.getTime() ?? 0)
+    );
+  }
+}

tests/test-team/helpers/s3-helper.ts

@@ -13,6 +13,7 @@
     "@playwright/test": "^1.51.1",
     "async-mutex": "^0.5.0",
     "aws-amplify": "^6.13.6",
+    "date-fns": "^4.1.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-playwright": "^2.2.2",

tests/test-team/package.json

@@ -242,7 +242,7 @@ test.describe('Event publishing - Letters', () => {
           }),
         })
       );
-    }).toPass({ timeout: 120_000, intervals: [1000, 3000, 5000, 10_000] });
+    }).toPass({ timeout: 90_000, intervals: [1000, 3000, 5000, 10_000] });
   });
 
   test('Expect Deleted.v1 event when deleting templates', async ({

tests/test-team/template-mgmt-event-tests/letter-templates.event.spec.ts

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "baseUrl": ".",
     "paths": {
       "@nhsdigital/nhs-notify-event-schemas-template-management": [
         "../../packages/event-schemas/src/index.ts"