Skip to content

Commit 216f508

Browse files
m-salaudeenchris-elliott-nhsd
m-salaudeen
authored and
chris-elliott-nhsd
committed
CCM-10429: PR comment fixes.
1 parent b461dbf commit 216f508

File tree

7 files changed

+242
-160
lines changed

7 files changed

+242
-160
lines changed

‎data-migration/user-transfer/src/user-transfer.ts‎

Lines changed: 79 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,21 @@ import {
77
retrieveAllTemplates,
88
updateItem,
99
} from '@/src/utils/ddb-utils';
10-
import { backupData } from '@/src/utils/backup-utils';
10+
import {
11+
backupData,
12+
backupToJSON,
13+
deleteJSONFile,
14+
readJSONFile,
15+
} from '@/src/utils/backup-utils';
1116
import { Parameters } from '@/src/utils/constants';
1217
import {
13-
findCognitoUser,
14-
getUserGroup,
18+
getUserGroupAndClientId,
1519
listCognitoUsers,
20+
UserData,
1621
} from './utils/cognito-utils';
17-
import {
18-
backupObject,
19-
copyObjects,
20-
deleteObjects,
21-
getItemObjects,
22-
} from './utils/s3-utils';
22+
import { backupObject, deleteObjects, handleS3Copy } from './utils/s3-utils';
23+
24+
const DRY_RUN = true;
2325

2426
function getParameters(): Parameters {
2527
return yargs(hideBin(process.argv))
@@ -32,85 +34,91 @@ function getParameters(): Parameters {
3234
type: 'string',
3335
demandOption: true,
3436
},
37+
accessKeyId: {
38+
type: 'string',
39+
demandOption: true,
40+
},
41+
secretAccessKey: {
42+
type: 'string',
43+
demandOption: true,
44+
},
45+
userPoolId: {
46+
type: 'string',
47+
demandOption: true,
48+
},
49+
sessionToken: {
50+
type: 'string',
51+
demandOption: true,
52+
},
53+
region: {
54+
type: 'string',
55+
demandOption: true,
56+
},
57+
flag: {
58+
type: 'string',
59+
demandOption: false,
60+
},
3561
})
3662
.parseSync();
3763
}
3864

39-
async function updateItems(
65+
async function getUserData(parameters: Parameters) {
66+
const usernames = await listCognitoUsers(parameters);
67+
if (!usernames || usernames.length === 0) {
68+
throw new Error('No users found');
69+
}
70+
const userGroupAndClientId = await getUserGroupAndClientId(
71+
usernames as string[],
72+
parameters
73+
);
74+
// download users to a json file
75+
await backupToJSON(userGroupAndClientId);
76+
}
77+
78+
async function migrateTemplatesAndS3Data(
4079
items: Record<string, AttributeValue>[],
4180
parameters: Parameters
4281
): Promise<void> {
43-
for (const item of items) {
44-
console.log(item.id.S, item.owner.S);
45-
46-
// Get owner id of this item
47-
const { owner, id, templateType, clientId } = item;
82+
const users: UserData[] = await readJSONFile('users.json');
4883

49-
//check if owner doesn't have CLIENT#
50-
if (owner.S && !owner.S?.startsWith('CLIENT#')) {
51-
// check the user in cognito, if it exist then pull the client id
52-
const cognitoUser = await findCognitoUser(owner.S);
84+
for (const user of users) {
85+
for (const item of items) {
86+
const { id } = item;
87+
if (id.S === user.userId) {
88+
// copy s3 data
89+
const sourceKey = await handleS3Copy(user, id.S as string, DRY_RUN);
5390

54-
// check and get user groups - this is used when migrating for production
55-
const userClientIdFromGroup = await getUserGroup({
56-
Username: cognitoUser?.username,
57-
UserPoolId: cognitoUser?.poolId,
58-
});
91+
// migrate templates
92+
await updateItem(item, parameters, user, DRY_RUN);
5993

60-
// resolve client id
61-
const newClientId = (userClientIdFromGroup ??
62-
cognitoUser?.clientIdAttr) as string;
94+
// delete previous template
95+
await deleteItem(item, parameters);
6396

64-
// check if this item has a client id, if yes check it matches the client id above. If it doesn't, throw error!
65-
if (item.clientId && item.clientId.S !== newClientId) {
66-
console.log({
67-
templateClientId: item.clientId,
68-
cognitoClientId: newClientId,
69-
});
70-
71-
throw new Error('Invalid ids');
72-
}
73-
// if it matches make the required swaps (clientId, createdby and updatedby)
74-
// if item doesn't have a client id then create one and do the above
75-
// update the item and delete the previous one
76-
77-
// migrate and update item
78-
await updateItem(item, parameters, newClientId);
79-
80-
// check if migration was successful
81-
82-
// delete migrated item
83-
await deleteItem(item, parameters);
84-
}
85-
86-
//
87-
if (templateType.S === 'LETTER') {
88-
//Retrieve item S3 object(s)
89-
const itemObjects = await getItemObjects(id.S as string);
90-
91-
//migrate to a new s3 location
92-
for (const itemObject of itemObjects) {
93-
await Promise.all([
94-
copyObjects(
95-
owner.S as string,
96-
itemObject['Key'],
97-
clientId.S as string
98-
),
99-
// delete previous objects
100-
deleteObjects(itemObject['Key']),
101-
]).then(() => console.log('Object moved'));
97+
// delete migrated s3 data
98+
deleteObjects(sourceKey);
10299
}
103100
}
104101
}
102+
103+
await deleteJSONFile('users.json');
104+
console.log('Migration completed successfully');
105105
}
106106

107107
export async function performTransfer() {
108108
const parameters = getParameters();
109+
110+
// if flag = user, then, get cognito users with their clientId and save to a json file
111+
if (parameters.flag && parameters.flag === 'user') {
112+
await getUserData(parameters);
113+
}
114+
115+
// retrieve and backup all templates
109116
const items = await retrieveAllTemplates(parameters);
110-
console.log({ items });
111-
console.log(await listCognitoUsers());
112-
// retrieve all user (id and clientId) in cognito and save to a JSON file
113-
// await backupData(items, parameters);
114-
// await backupObject(parameters);
115-
// await updateItems(items, parameters);
117+
await backupData(items, parameters);
118+
119+
// retrieve and backup all S3 data
120+
await backupObject(parameters);
121+
122+
// Migrate
123+
await migrateTemplatesAndS3Data(items, parameters);
116124
}

‎data-migration/user-transfer/src/utils/backup-utils.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { AttributeValue } from '@aws-sdk/client-dynamodb';
22
import { Parameters } from '@/src/utils/constants';
33
import { getAccountId } from '@/src/utils/sts-utils';
44
import { writeJsonToFile } from '@/src/utils/s3-utils';
5+
import fs from 'node:fs';
6+
import { UserData } from './cognito-utils';
57

68
export async function backupData(
79
items: Record<string, AttributeValue>[],
@@ -21,3 +23,34 @@ export async function backupData(
2123
await writeJsonToFile(filePath, JSON.stringify(items), bucketName);
2224
console.log(`Backed up data to s3://${bucketName}/${filePath}`);
2325
}
26+
27+
export async function backupToJSON(userData: UserData[]) {
28+
const userDataJSON = JSON.stringify(userData);
29+
30+
fs.writeFile('users.json', userDataJSON, (err) => {
31+
if (err) {
32+
console.log('Error writing file:', err);
33+
} else {
34+
console.log('Successfully wrote file');
35+
}
36+
});
37+
}
38+
39+
export async function readJSONFile(filePath: string) {
40+
// eslint-disable-next-line security/detect-non-literal-fs-filename
41+
const users = fs.readFileSync(filePath, 'utf8');
42+
43+
return JSON.parse(users);
44+
}
45+
46+
export async function deleteJSONFile(filePath: string) {
47+
fs.access(filePath, (error) => {
48+
if (error) {
49+
console.error('Error occurred:', error);
50+
} else {
51+
// eslint-disable-next-line security/detect-non-literal-fs-filename
52+
fs.unlinkSync(filePath);
53+
console.log('File deleted successfully');
54+
}
55+
});
56+
}

‎data-migration/user-transfer/src/utils/cognito-utils.ts‎

Lines changed: 56 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -2,80 +2,77 @@ import {
22
CognitoIdentityProviderClient,
33
ListUsersCommand,
44
AdminListGroupsForUserCommand,
5-
AdminListGroupsForUserCommandInput,
6-
UserType,
75
GroupType,
86
} from '@aws-sdk/client-cognito-identity-provider';
9-
import { ListBackupsCommandOutput } from '@aws-sdk/client-dynamodb';
10-
import { fromIni } from '@aws-sdk/credential-providers';
7+
import { Parameters } from './constants';
118

12-
const COGNITO_PROFILE = process.env.COGNITO_ACCOUNT;
13-
const cognito = new CognitoIdentityProviderClient({
14-
region: process.env.REGION,
15-
credentials: fromIni({ profile: COGNITO_PROFILE }),
16-
});
17-
const USER_POOL_ID = process.env.USER_POOL_ID;
9+
export type UserData = {
10+
userId: string;
11+
clientId: string;
12+
};
1813

19-
interface CognitoUserBasics {
20-
username: string;
21-
sub?: string;
22-
clientIdAttr?: string;
23-
poolId: string;
24-
user?: UserType;
25-
}
26-
27-
export async function listCognitoUsers(): Promise<
28-
ListBackupsCommandOutput | undefined
29-
> {
14+
export async function listCognitoUsers(
15+
parameters: Parameters
16+
): Promise<Array<string> | undefined> {
17+
let usernames: string[] = [];
18+
const { region, accessKeyId, secretAccessKey, userPoolId, sessionToken } =
19+
parameters;
20+
const cognito = new CognitoIdentityProviderClient({
21+
region,
22+
credentials: {
23+
accessKeyId,
24+
secretAccessKey,
25+
sessionToken,
26+
},
27+
});
3028
const command = new ListUsersCommand({
31-
UserPoolId: 'eu-west-2_lGFnZO7vx',
29+
UserPoolId: userPoolId,
3230
});
3331

3432
const response = await cognito.send(command);
35-
return response;
36-
}
3733

38-
export async function findCognitoUser(
39-
ownerId: string
40-
): Promise<CognitoUserBasics | undefined> {
41-
console.log('owner', ownerId);
42-
const listUser = new ListUsersCommand({
43-
UserPoolId: USER_POOL_ID,
44-
Filter: `"sub"="${ownerId}"`,
45-
});
46-
const res = await cognito.send(listUser);
47-
const user = res.Users?.[0];
48-
if (user) {
49-
const sub = user.Attributes?.find((a) => a.Name === 'sub')?.Value;
50-
const clientIdAttr = user.Attributes?.find(
51-
(a) => a.Name === 'custom:sbx_client_id' // this would be removed when migrating for production
52-
)?.Value;
53-
return {
54-
username: user.Username!,
55-
sub,
56-
clientIdAttr,
57-
poolId: USER_POOL_ID as string,
58-
user,
59-
};
34+
if (response.Users) {
35+
usernames = response.Users.map((user) => user.Username as string);
36+
return usernames;
6037
}
38+
6139
return undefined;
6240
}
6341

64-
export async function getUserGroup(
65-
input: AdminListGroupsForUserCommandInput
66-
): Promise<string | undefined> {
67-
const { Username, UserPoolId } = input;
42+
export async function getUserGroupAndClientId(
43+
usernames: string[],
44+
parameters: Parameters
45+
): Promise<UserData[]> {
46+
const userIdAndClientId: UserData[] = [];
47+
const { region, accessKeyId, secretAccessKey, userPoolId, sessionToken } =
48+
parameters;
49+
const cognito = new CognitoIdentityProviderClient({
50+
region,
51+
credentials: {
52+
accessKeyId,
53+
secretAccessKey,
54+
sessionToken,
55+
},
56+
});
6857

69-
const userGroups = await cognito.send(
70-
new AdminListGroupsForUserCommand({
71-
UserPoolId,
72-
Username,
73-
})
74-
);
58+
for (const username of usernames) {
59+
const userGroups = await cognito.send(
60+
new AdminListGroupsForUserCommand({
61+
UserPoolId: userPoolId,
62+
Username: username,
63+
})
64+
);
65+
66+
if (userGroups.Groups && userGroups.Groups?.length > 0) {
67+
const getClientId = await getUserClientId(userGroups.Groups);
68+
userIdAndClientId.push({
69+
userId: username,
70+
clientId: getClientId as string,
71+
});
72+
}
73+
}
7574

76-
return userGroups.Groups && userGroups.Groups?.length === 0
77-
? undefined
78-
: await getUserClientId(userGroups.Groups);
75+
return userIdAndClientId;
7976
}
8077

8178
async function getUserClientId(

‎data-migration/user-transfer/src/utils/constants.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
export type Parameters = {
2-
sourceOwner?: string;
3-
destinationOwner?: string;
42
environment: string;
53
component: string;
4+
accessKeyId: string;
5+
secretAccessKey: string;
6+
userPoolId: string;
7+
region: string;
8+
sessionToken: string;
9+
flag?: string;
610
};

0 commit comments

Comments
 (0)