CCM-10429: backup data

Author: bhansell1

data-migration/user-transfer/src/__tests__/utils/backup-utils.test.ts

@@ -1,6 +1,6 @@
 import { backupData } from '@/src/utils/backup-utils';
 import { getAccountId } from '@/src/utils/sts-utils';
-import { writeJsonToFile } from '@/src/utils/s3-utils';
+import { writeFile } from '@/src/utils/s3-utils';
 
 const mockItem1 = {
   templateType: {
@@ -45,7 +45,7 @@ describe('backup-utils', () => {
       jest
         .mocked(getAccountId)
         .mockImplementation(() => Promise.resolve('000000000000'));
-      const mockedWriteJsonToFile = jest.mocked(writeJsonToFile);
+      const mockedWriteJsonToFile = jest.mocked(writeFile);
 
       const testParameters = {
         destinationOwner: 'def-456',
@@ -73,7 +73,7 @@ describe('backup-utils', () => {
 
     test('should no-op the backup when no items have been found', async () => {
       // arrange
-      const mockedWriteJsonToFile = jest.mocked(writeJsonToFile);
+      const mockedWriteJsonToFile = jest.mocked(writeFile);
 
       const testParameters = {
         destinationOwner: 'def-456',

data-migration/user-transfer/src/__tests__/utils/s3-utils.test.ts

@@ -1,4 +1,4 @@
-import { writeJsonToFile } from '@/src/utils/s3-utils';
+import { writeFile } from '@/src/utils/s3-utils';
 import { S3Client } from '@aws-sdk/client-s3';
 
 jest.mock('@aws-sdk/client-s3', () => ({
@@ -17,7 +17,7 @@ describe('s3-utils', () => {
       const testFilePath = '/test/file/path.json';
 
       // act
-      await writeJsonToFile(testFilePath, testContent, testBucketName);
+      await writeFile(testFilePath, testContent, testBucketName);
 
       // assert
       expect(sendSpy).toHaveBeenCalledTimes(1);

data-migration/user-transfer/src/handler.ts

@@ -4,16 +4,23 @@ import fs from 'node:fs';
 import path from 'node:path';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import { MigrationPlan } from './utils/constants';
-import { print } from './utils/log';
-import { MigrationHandler } from './utils/migration-handler';
+import { UserTransferPlan, UserTransferPlanItem } from './utils/types';
+import { print } from './utils/log-utils';
+import { UserTransfer } from './utils/user-transfer';
+import { getTemplates } from './utils/ddb-utils';
+import { backupBucketName, backupData } from './utils/backup-utils';
+import { transferFileToNewBucket, writeFile } from './utils/s3-utils';
 
 const params = yargs(hideBin(process.argv))
   .options({
     file: {
       type: 'string',
       demandOption: true,
     },
+    environment: {
+      type: 'string',
+      demandOption: true,
+    },
     dryRun: {
       type: 'boolean',
       default: true,
@@ -31,20 +38,66 @@ function writeLocal(filename: string, data: string) {
   });
 }
 
+async function backup(
+  tableName: string,
+  sourceBucket: string,
+  backupBucket: string,
+  migrations: UserTransferPlanItem[]
+) {
+  const keys = migrations.map((r) => ({
+    id: r.templateId,
+    owner: r.ddb.owner.from,
+  }));
+
+  const files = migrations.flatMap((r) => r.s3.files.flatMap((a) => a.from));
+
+  const data = await getTemplates(tableName, keys);
+
+  if (data.length !== migrations.length) {
+    throw new Error(
+      `Only retrieved ${data.length}/${migrations.length} migrations`
+    );
+  }
+
+  const { name } = path.parse(params.file);
+
+  await backupData(
+    data,
+    backupBucket,
+    `ownership-transfer/${params.environment}/${name}`
+  );
+
+  await Promise.all(
+    files.map((file) =>
+      transferFileToNewBucket(
+        sourceBucket,
+        backupBucket,
+        file,
+        `ownership-transfer/${params.environment}/${name}/${file}`
+      )
+    )
+  );
+}
+
 async function migrate() {
+  const backupBucket = await backupBucketName();
+
   const input = JSON.parse(
     fs.readFileSync(params.file, 'utf8')
-  ) as MigrationPlan;
+  ) as UserTransferPlan;
 
-  const output: MigrationPlan = {
+  const output: UserTransferPlan = {
     ...input,
-    run: input.run + 1,
   };
 
   const migrations = input.migrate.plans.filter((r) => r.status === 'migrate');
 
   print(`Total migrations: ${migrations.length}`);
 
+  if (!params.dryRun) {
+    await backup(input.tableName, input.bucketName, backupBucket, migrations);
+  }
+
   for (let i = 0; i < migrations.length; i++) {
     const migration = migrations[i];
 
@@ -53,14 +106,14 @@ async function migrate() {
     );
 
     if (idx === -1) {
-      print('Skipping: Unable to locate index for ${templateId}');
+      print(`Skipping: Unable to locate index for ${migration.templateId}`);
       continue;
     }
 
     print(`Progress: ${i + 1}/${migrations.length}`);
     print(`Processing: ${migration.templateId}`);
 
-    const result = await MigrationHandler.apply(migration, {
+    const result = await UserTransfer.apply(migration, {
       bucketName: input.bucketName,
       tableName: input.tableName,
       dryRun: params.dryRun,
@@ -78,11 +131,17 @@ async function migrate() {
 
   const { dir, name, ext } = path.parse(params.file);
 
-  const fileName = `${name}-${params.dryRun ? 'dryrun-' : 'run-'}${output.run}${ext}`;
+  const fileName = `${name}-${params.dryRun ? 'dryrun' : ''}${ext}`;
 
   const data = JSON.stringify(output);
 
+  const filePath = `ownership-transfer/${params.environment}/${name}/${fileName}`;
+
   writeLocal(path.join(dir, fileName), data);
+
+  await writeFile(filePath, data, backupBucket);
+
+  print(`Plan written to s3://${backupBucket}/${filePath}`);
 }
 
 migrate()

data-migration/user-transfer/src/migrate.ts

@@ -1,16 +1,14 @@
 /* eslint-disable unicorn/no-process-exit */
 /* eslint-disable unicorn/prefer-top-level-await */
 import { CognitoRepository } from './utils/cognito-repository';
-import { retrieveAllTemplatesV2 } from './utils/ddb-utils';
-import {
-  listItemObjectsWithPaginator,
-  writeJsonToFile as writeRemote,
-} from './utils/s3-utils';
-import { MigrationHandler } from './utils/migration-handler';
+import { listAllTemplates } from './utils/ddb-utils';
+import { listAllFiles, writeFile } from './utils/s3-utils';
+import { UserTransfer } from './utils/user-transfer';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import { getAccountId } from './utils/sts-utils';
-import { print } from './utils/log';
+import { print } from './utils/log-utils';
+import { backupBucketName } from './utils/backup-utils';
 
 const params = yargs(hideBin(process.argv))
   .options({
@@ -52,7 +50,7 @@ const params = yargs(hideBin(process.argv))
   })
   .parseSync();
 
-const cognitoCrawler = new CognitoRepository(
+const cognitoRepository = new CognitoRepository(
   params.userPoolId,
   params.component === 'app'
     ? {
@@ -65,12 +63,13 @@ const cognitoCrawler = new CognitoRepository(
 
 async function getConfig() {
   const accountId = await getAccountId();
+  const backupBucket = await backupBucketName();
 
   return {
     accountId,
     templateTableName: `nhs-notify-${params.environment}-${params.component}-api-templates`,
     templatesS3InternalBucketName: `nhs-notify-${accountId}-eu-west-2-${params.environment}-${params.component}-internal`,
-    templatesS3BackupBucketName: `nhs-notify-${accountId}-eu-west-2-main-acct-migration-backup`,
+    templatesS3BackupBucketName: backupBucket,
   };
 }
 
@@ -82,15 +81,13 @@ async function plan() {
     templatesS3BackupBucketName,
   } = await getConfig();
 
-  const users = await cognitoCrawler.getAllUsers();
+  const users = await cognitoRepository.getAllUsers();
 
-  const templates = await retrieveAllTemplatesV2(templateTableName);
+  const templates = await listAllTemplates(templateTableName);
 
-  const files = await listItemObjectsWithPaginator(
-    templatesS3InternalBucketName
-  );
+  const files = await listAllFiles(templatesS3InternalBucketName);
 
-  const transferPlan = await MigrationHandler.plan(
+  const transferPlan = await UserTransfer.plan(
     users,
     {
       tableName: templateTableName,
@@ -102,13 +99,13 @@ async function plan() {
     }
   );
 
-  const fileName = `transfer-plan-${accountId}-${params.environment}-${params.component}-${Date.now()}.json`;
+  const fileName = `transfer-plan-${accountId}-${params.environment}-${params.component}-${Date.now()}`;
 
   const data = JSON.stringify(transferPlan);
 
-  const filePath = `ownership-transfer/${params.environment}/${fileName}`;
+  const filePath = `ownership-transfer/${params.environment}/${fileName}/${fileName}.json`;
 
-  await writeRemote(filePath, data, templatesS3BackupBucketName);
+  await writeFile(filePath, data, templatesS3BackupBucketName);
 
   print(`Plan written to s3://${templatesS3InternalBucketName}/${filePath}`);
 }