CCM-10057: backup to S3

Author: mark-r-bjss

data-migration/user-transfer/package.json

@@ -1,6 +1,8 @@
 {
   "dependencies": {
     "@aws-sdk/client-dynamodb": "^3.806.0",
+    "@aws-sdk/client-s3": "^3.810.0",
+    "@aws-sdk/client-sts": "^3.810.0",
     "yargs": "^17.7.2"
   },
   "description": "Transfers template ownership from one user to another",

data-migration/user-transfer/src/__tests__/utils/backup-utils.test.ts

@@ -1,5 +1,6 @@
 import { backupData } from '@/src/utils/backup-utils';
-import * as fileSystem from 'node:fs';
+import { getAccountId } from '@/src/utils/sts-utils';
+import { writeJsonToFile } from '@/src/utils/s3-utils';
 
 const mockItem1 = {
   templateType: {
@@ -31,25 +32,20 @@ const mockItem2 = {
   },
 };
 
-jest.mock('node:fs', () => ({
-  __esModule: true,
-  ...jest.requireActual('node:fs'),
-}));
-
-const writeFileSyncSpy = jest.spyOn(fileSystem, 'writeFileSync');
-const existsSyncSpy = jest.spyOn(fileSystem, 'existsSync');
-const mkdirSyncSpy = jest.spyOn(fileSystem, 'mkdirSync');
+jest.mock('@/src/utils/sts-utils');
+jest.mock('@/src/utils/s3-utils');
 
 describe('backup-utils', () => {
   describe('backupData', () => {
-    test('should backup data', () => {
+    test('should backup data', async () => {
       // arrange
       jest.useFakeTimers();
       jest.setSystemTime(new Date('2025-05-13'));
 
-      writeFileSyncSpy.mockImplementation(() => {});
-      existsSyncSpy.mockImplementation(() => false);
-      mkdirSyncSpy.mockImplementation(() => '');
+      jest
+        .mocked(getAccountId)
+        .mockImplementation(() => Promise.resolve('000000000000'));
+      const mockedWriteJsonToFile = jest.mocked(writeJsonToFile);
 
       const testParameters = {
         destinationOwner: 'def-456',
@@ -59,24 +55,25 @@ describe('backup-utils', () => {
 
       const mockItems = [mockItem1, mockItem2];
       const expectedBackupFilePath =
-        './backups/usr_tsfr-2025-05-13T00:00:00.000Z-env-testenv-src-abc-123-dest-def-456.json';
+        'user-transfer/testenv/2025_05_13_00_00_00_000Z-source-abc-123-destination-def-456.json';
+      const expectedBucketName =
+        'nhs-notify-000000000000-eu-west-2-main-acct-migration-backup';
+      const expectedContent = JSON.stringify(mockItems);
 
       // act
-      backupData(mockItems, testParameters);
+      await backupData(mockItems, testParameters);
 
       // assert
-      expect(mkdirSyncSpy).toHaveBeenCalledWith('./backups');
-      expect(writeFileSyncSpy).toHaveBeenCalledWith(
+      expect(mockedWriteJsonToFile).toHaveBeenCalledWith(
         expectedBackupFilePath,
-        JSON.stringify(mockItems)
+        expectedContent,
+        expectedBucketName
       );
     });
 
-    test('should handle existing backup dir', () => {
+    test('should no-op the backup when no items have been found', async () => {
       // arrange
-      writeFileSyncSpy.mockImplementation(() => {});
-      existsSyncSpy.mockImplementation(() => true);
-      mkdirSyncSpy.mockImplementation(() => '');
+      const mockedWriteJsonToFile = jest.mocked(writeJsonToFile);
 
       const testParameters = {
         destinationOwner: 'def-456',
@@ -85,10 +82,10 @@ describe('backup-utils', () => {
       };
 
       // act
-      backupData([], testParameters);
+      await backupData([], testParameters);
 
       // assert
-      expect(mkdirSyncSpy).not.toHaveBeenCalled();
+      expect(mockedWriteJsonToFile).not.toHaveBeenCalled();
     });
   });
 

data-migration/user-transfer/src/__tests__/utils/s3-utils.test.ts

@@ -0,0 +1,36 @@
+import { writeJsonToFile } from '@/src/utils/s3-utils';
+import { S3Client } from '@aws-sdk/client-s3';
+
+jest.mock('@aws-sdk/client-s3', () => ({
+  ...jest.requireActual('@aws-sdk/client-s3'),
+}));
+
+describe('s3-utils', () => {
+  describe('writeJsonToFile', () => {
+    test('should write JSON to an S3 object', async () => {
+      // arrange
+      const sendSpy = jest.spyOn(S3Client.prototype, 'send');
+      sendSpy.mockImplementation(() => {});
+
+      const testContent = '[{"test":"content"}]';
+      const testBucketName = 'test-bucket-name';
+      const testFilePath = '/test/file/path.json';
+
+      // act
+      await writeJsonToFile(testFilePath, testContent, testBucketName);
+
+      // assert
+      expect(sendSpy).toHaveBeenCalledTimes(1);
+      expect(sendSpy).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: {
+            Body: testContent,
+            Bucket: testBucketName,
+            ContentType: 'application/json',
+            Key: testFilePath,
+          },
+        })
+      );
+    });
+  });
+});

data-migration/user-transfer/src/__tests__/utils/sts-utils.test.ts

@@ -0,0 +1,44 @@
+import { getAccountId } from '@/src/utils/sts-utils';
+import { STSClient } from '@aws-sdk/client-sts';
+
+jest.mock('@aws-sdk/client-sts', () => ({
+  ...jest.requireActual('@aws-sdk/client-sts'),
+}));
+
+describe('sts-utils', () => {
+  describe('getAccountId', () => {
+    test('should get account ID from current auth context', async () => {
+      // arrange
+      const sendSpy = jest.spyOn(STSClient.prototype, 'send');
+      sendSpy.mockImplementation(() => ({
+        Account: '000000000000',
+      }));
+
+      // act
+      const result = await getAccountId();
+
+      // assert
+      expect(result).toBe('000000000000');
+    });
+
+    test('should reject missing account ID', async () => {
+      // arrange
+      const sendSpy = jest.spyOn(STSClient.prototype, 'send');
+      sendSpy.mockImplementation(() => ({}));
+
+      // act
+      let caughtError;
+      try {
+        await getAccountId();
+      } catch (error) {
+        caughtError = error;
+      }
+
+      // assert
+      expect(caughtError).toBeTruthy();
+      expect((caughtError as Error).message).toBe(
+        'Unable to get account ID from caller'
+      );
+    });
+  });
+});

data-migration/user-transfer/src/user-transfer.ts

@@ -4,12 +4,7 @@ import { hideBin } from 'yargs/helpers';
 import { AttributeValue } from '@aws-sdk/client-dynamodb';
 import { retrieveTemplates, updateItem } from '@/src/utils/ddb-utils';
 import { backupData } from '@/src/utils/backup-utils';
-
-export type Parameters = {
-  sourceOwner: string;
-  destinationOwner: string;
-  environment: string;
-};
+import { Parameters } from '@/src/utils/constants';
 
 function getParameters(): Parameters {
   return yargs(hideBin(process.argv))
@@ -46,6 +41,6 @@ async function updateItems(
 export async function performTransfer() {
   const parameters = getParameters();
   const items = await retrieveTemplates(parameters);
-  backupData(items, parameters);
+  await backupData(items, parameters);
   await updateItems(items, parameters);
 }

data-migration/user-transfer/src/utils/backup-utils.ts

@@ -1,24 +1,24 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 import { AttributeValue } from '@aws-sdk/client-dynamodb';
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { Parameters } from '@/src/utils/constants';
+import { getAccountId } from '@/src/utils/sts-utils';
+import { writeJsonToFile } from '@/src/utils/s3-utils';
 
-const backupDir = './backups';
-
-function createBackupDir() {
-  if (!existsSync(backupDir)) {
-    mkdirSync(backupDir);
-  }
-}
-
-export function backupData(
+export async function backupData(
   items: Record<string, AttributeValue>[],
   parameters: Parameters
-) {
-  const { environment, sourceOwner, destinationOwner } = parameters;
-  createBackupDir();
+): Promise<void> {
   console.log(`Found ${items.length} results`);
-  const fileName = `usr_tsfr-${new Date().toISOString()}-env-${environment}-src-${sourceOwner}-dest-${destinationOwner}.json`;
-  writeFileSync(`${backupDir}/${fileName}`, JSON.stringify(items));
-  console.log(`Backed up data locally to ${fileName}`);
+  if (items.length <= 0) {
+    return;
+  }
+
+  const { environment, sourceOwner, destinationOwner } = parameters;
+  const accountId = await getAccountId();
+  const bucketName = `nhs-notify-${accountId}-eu-west-2-main-acct-migration-backup`;
+
+  const timestamp = new Date().toISOString().replaceAll(/[.:T-]/g, '_');
+  const filePath = `user-transfer/${environment}/${timestamp}-source-${sourceOwner}-destination-${destinationOwner}.json`;
+  await writeJsonToFile(filePath, JSON.stringify(items), bucketName);
+  console.log(`Backed up data to s3://${bucketName}/${filePath}`);
 }

data-migration/user-transfer/src/utils/s3-utils.ts

@@ -0,0 +1,26 @@
+import {
+  PutObjectCommand,
+  PutObjectCommandOutput,
+  S3Client,
+} from '@aws-sdk/client-s3';
+
+const s3Client = new S3Client({
+  region: process.env.REGION,
+  retryMode: 'standard',
+  maxAttempts: 10,
+});
+
+export async function writeJsonToFile(
+  path: string,
+  content: string,
+  bucket: string
+): Promise<PutObjectCommandOutput> {
+  return s3Client.send(
+    new PutObjectCommand({
+      Bucket: bucket,
+      Key: path,
+      Body: content,
+      ContentType: 'application/json',
+    })
+  );
+}

data-migration/user-transfer/src/utils/sts-utils.ts

@@ -0,0 +1,12 @@
+import { GetCallerIdentityCommand, STSClient } from '@aws-sdk/client-sts';
+
+const stsClient = new STSClient({ region: process.env.REGION });
+
+export async function getAccountId(): Promise<string> {
+  const callerIdentity = await stsClient.send(new GetCallerIdentityCommand());
+  const accountId = callerIdentity.Account;
+  if (!accountId) {
+    throw new Error('Unable to get account ID from caller');
+  }
+  return accountId;
+}