CCM-10429: refactor

Author: bhansell1

data-migration/user-transfer/.gitignore

@@ -0,0 +1 @@
+migrations/*

data-migration/user-transfer/src/migrate.ts

@@ -0,0 +1,90 @@
+/* eslint-disable security/detect-non-literal-fs-filename */
+/* eslint-disable unicorn/prefer-top-level-await */
+import fs from 'node:fs';
+import path from 'node:path';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { MigrationPlan } from './utils/constants';
+import { print } from './utils/log';
+import { MigrationHandler } from './utils/migration-handler';
+
+const params = yargs(hideBin(process.argv))
+  .options({
+    file: {
+      type: 'string',
+      demandOption: true,
+    },
+    dryRun: {
+      type: 'boolean',
+      default: true,
+    },
+  })
+  .parseSync();
+
+function writeLocal(filename: string, data: string) {
+  fs.writeFile(filename, data, (err) => {
+    if (err) {
+      console.log(`Error writing file: ${filename}`, err);
+    } else {
+      console.log(`Successfully wrote ${filename}`);
+    }
+  });
+}
+
+async function migrate() {
+  const input = JSON.parse(
+    fs.readFileSync(params.file, 'utf8')
+  ) as MigrationPlan;
+
+  const output: MigrationPlan = {
+    ...input,
+    run: input.run + 1,
+  };
+
+  const migrations = input.migrate.plans.filter((r) => r.status === 'migrate');
+
+  print(`Total migrations: ${migrations.length}`);
+
+  for (let i = 0; i < migrations.length; i++) {
+    const migration = migrations[i];
+
+    const idx = output.migrate.plans.findIndex(
+      (r) => r.templateId === migration.templateId
+    );
+
+    if (idx === -1) {
+      print('Skipping: Unable to locate index for ${templateId}');
+      continue;
+    }
+
+    print(`Progress: ${i + 1}/${migrations.length}`);
+    print(`Processing: ${migration.templateId}`);
+
+    const result = await MigrationHandler.apply(migration, {
+      bucketName: input.bucketName,
+      tableName: input.tableName,
+      dryRun: params.dryRun,
+    });
+
+    output.migrate.plans[idx] = {
+      ...output.migrate.plans[idx],
+      stage: result.stage,
+      status: result.success ? 'success' : 'failed',
+      reason: JSON.stringify(result.reasons),
+    };
+
+    print(`Result: success - [${result.success}]`);
+  }
+
+  const { dir, name, ext } = path.parse(params.file);
+
+  const fileName = `${name}-${params.dryRun ? 'dryrun-' : 'run-'}${output.run}${ext}`;
+
+  const data = JSON.stringify(output);
+
+  writeLocal(path.join(dir, fileName), data);
+}
+
+migrate()
+  .then(() => console.log('finished'))
+  .catch((error) => console.error(error));

data-migration/user-transfer/src/plan.ts

@@ -0,0 +1,119 @@
+/* eslint-disable unicorn/no-process-exit */
+/* eslint-disable unicorn/prefer-top-level-await */
+import { CognitoRepository } from './utils/cognito-repository';
+import { retrieveAllTemplatesV2 } from './utils/ddb-utils';
+import {
+  listItemObjectsWithPaginator,
+  writeJsonToFile as writeRemote,
+} from './utils/s3-utils';
+import { MigrationHandler } from './utils/migration-handler';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { getAccountId } from './utils/sts-utils';
+import { print } from './utils/log';
+
+const params = yargs(hideBin(process.argv))
+  .options({
+    environment: {
+      type: 'string',
+      demandOption: true,
+    },
+    component: {
+      type: 'string',
+      default: 'app',
+      options: ['sbx', 'app'],
+    },
+    userPoolId: {
+      type: 'string',
+      demandOption: true,
+    },
+    iamAccessKeyId: {
+      type: 'string',
+    },
+    iamSecretAccessKey: {
+      type: 'string',
+    },
+    iamSessionToken: {
+      type: 'string',
+    },
+  })
+  .check((argv) => {
+    const iamSet =
+      !!argv.iamAccessKeyId &&
+      !!argv.iamSecretAccessKey &&
+      !!argv.iamSessionToken;
+
+    if (argv.component === 'app' && !iamSet) {
+      throw new Error(
+        'iamAccessKeyId, iamSecretAccessKey, iamSessionToken - must be set when using component: app'
+      );
+    }
+    return true;
+  })
+  .parseSync();
+
+const cognitoCrawler = new CognitoRepository(
+  params.userPoolId,
+  params.component === 'app'
+    ? {
+        accessKeyId: params.iamAccessKeyId!,
+        secretAccessKey: params.iamSecretAccessKey!,
+        sessionToken: params.iamSessionToken!,
+      }
+    : undefined
+);
+
+async function getConfig() {
+  const accountId = await getAccountId();
+
+  return {
+    accountId,
+    templateTableName: `nhs-notify-${params.environment}-${params.component}-api-templates`,
+    templatesS3InternalBucketName: `nhs-notify-${accountId}-eu-west-2-${params.environment}-${params.component}-internal`,
+    templatesS3BackupBucketName: `nhs-notify-${accountId}-eu-west-2-main-acct-migration-backup`,
+  };
+}
+
+async function plan() {
+  const {
+    accountId,
+    templateTableName,
+    templatesS3InternalBucketName,
+    templatesS3BackupBucketName,
+  } = await getConfig();
+
+  const users = await cognitoCrawler.getAllUsers();
+
+  const templates = await retrieveAllTemplatesV2(templateTableName);
+
+  const files = await listItemObjectsWithPaginator(
+    templatesS3InternalBucketName
+  );
+
+  const transferPlan = await MigrationHandler.plan(
+    users,
+    {
+      tableName: templateTableName,
+      templates,
+    },
+    {
+      bucketName: templatesS3InternalBucketName,
+      files,
+    }
+  );
+
+  const fileName = `transfer-plan-${accountId}-${params.environment}-${params.component}-${Date.now()}.json`;
+
+  const data = JSON.stringify(transferPlan);
+
+  const filePath = `ownership-transfer/${params.environment}/${fileName}`;
+
+  await writeRemote(filePath, data, templatesS3BackupBucketName);
+
+  print(`Plan written to s3://${templatesS3InternalBucketName}/${filePath}`);
+}
+
+plan().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

data-migration/user-transfer/src/utils/cognito-repository.ts

@@ -0,0 +1,123 @@
+/* eslint-disable sonarjs/no-commented-code */
+import {
+  CognitoIdentityProviderClient,
+  ListUsersCommand,
+  ListUsersCommandInput,
+  AdminListGroupsForUserCommand,
+  UserType,
+  GroupType,
+} from '@aws-sdk/client-cognito-identity-provider';
+import { print } from './log';
+
+export type UserData = {
+  username: string;
+  /*
+   * extracted from the user's group starting with 'client:'
+   */
+  clientId: string;
+  /*
+   * extracted from the user's 'sub' attribute
+   */
+  userId: string;
+};
+
+export class CognitoRepository {
+  private client: CognitoIdentityProviderClient;
+  private userPoolId: string;
+
+  constructor(
+    userPoolId: string,
+    credentials?: {
+      accessKeyId: string;
+      secretAccessKey: string;
+      sessionToken: string;
+    }
+  ) {
+    this.userPoolId = userPoolId;
+
+    this.client = new CognitoIdentityProviderClient({
+      region: 'eu-west-2',
+      credentials,
+    });
+  }
+
+  async getAllUsers(): Promise<UserData[]> {
+    const users: UserData[] = [];
+
+    let paginationToken: string | undefined;
+
+    try {
+      do {
+        const params: ListUsersCommandInput = {
+          UserPoolId: this.userPoolId,
+          Limit: 60, // Max allowed by AWS
+          PaginationToken: paginationToken,
+        };
+
+        const command = new ListUsersCommand(params);
+
+        const response = await this.client.send(command);
+
+        if (response.Users) {
+          const userBatch = await Promise.all(
+            response.Users.map((user) => this.processUser(user))
+          );
+
+          users.push(...userBatch.filter((r) => r !== undefined));
+        }
+
+        paginationToken = response.PaginationToken;
+      } while (paginationToken);
+
+      return users;
+    } catch (error) {
+      print('Failed: fetching users');
+      throw error;
+    }
+  }
+
+  private async processUser(user: UserType): Promise<UserData | undefined> {
+    const username = user.Username!;
+
+    const userId = user.Attributes?.find(({ Name }) => Name === 'sub')?.Value;
+
+    const clientId = await this.getClientIdFromGroups(username);
+
+    if (!clientId || !userId) {
+      print(`Ignoring User: ${username}. No client or sub assigned`);
+      return;
+    }
+
+    return {
+      username,
+      clientId,
+      userId,
+    };
+  }
+
+  private async getClientIdFromGroups(
+    username: string
+  ): Promise<string | undefined> {
+    try {
+      const command = new AdminListGroupsForUserCommand({
+        UserPoolId: this.userPoolId,
+        Username: username,
+      });
+
+      const response = await this.client.send(command);
+
+      if (!response.Groups) {
+        return;
+      }
+
+      const clientGroup = response.Groups.find((group: GroupType) =>
+        group.GroupName?.startsWith('client:')
+      );
+
+      return clientGroup?.GroupName?.split(':')[1];
+    } catch (error) {
+      print(`Failed: to get groups for user ${username}`);
+      throw error;
+    }
+  }
+}

data-migration/user-transfer/src/utils/constants.ts

@@ -8,3 +8,56 @@ export type Parameters = {
   sessionToken: string;
   flag?: string;
 };
+
+export type Template = {
+  id: string;
+  owner: string;
+};
+
+type ToFrom = {
+  from: string;
+  to: string;
+};
+
+export type MigrationStage =
+  | 's3:copy'
+  | 'ddb:transfer'
+  | 's3:delete'
+  | 'initial'
+  | 'finished';
+
+export type MigrationStatus = 'success' | 'failed' | 'migrate';
+
+export type MigrationPlanItem = {
+  templateId: string;
+  status: MigrationStatus;
+  stage: MigrationStage;
+  reason?: string;
+  ddb: {
+    owner: ToFrom;
+  };
+  s3: {
+    files: ToFrom[];
+  };
+};
+
+export type MigrationPlan = {
+  total: number;
+  tableName: string;
+  bucketName: string;
+  run: number;
+  migrate: {
+    count: number;
+    plans: MigrationPlanItem[];
+  };
+  orphaned: {
+    count: number;
+    templateIds: string[];
+  };
+};
+
+export type MigrationPlanItemResult = {
+  success: boolean;
+  stage: MigrationStage;
+  reasons?: string[];
+};