Skip to content

Commit c76441d

Browse files
m-salaudeenchris-elliott-nhsd
m-salaudeen
authored and
chris-elliott-nhsd
committed
CCM-10429: Templates and S3 migration
1 parent eeb91d4 commit c76441d

File tree

8 files changed

+343
-47
lines changed

8 files changed

+343
-47
lines changed

‎data-migration/user-transfer/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"dependencies": {
3+
"@aws-sdk/client-cognito-identity-provider": "^3.879.0",
34
"@aws-sdk/client-dynamodb": "^3.864.0",
45
"@aws-sdk/client-s3": "^3.864.0",
56
"@aws-sdk/client-sts": "^3.864.0",

‎data-migration/user-transfer/src/user-transfer.ts‎

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,24 @@
22
import yargs from 'yargs/yargs';
33
import { hideBin } from 'yargs/helpers';
44
import { AttributeValue } from '@aws-sdk/client-dynamodb';
5-
import { retrieveTemplates, updateItem } from '@/src/utils/ddb-utils';
5+
import {
6+
deleteItem,
7+
retrieveAllTemplates,
8+
updateItem,
9+
} from '@/src/utils/ddb-utils';
610
import { backupData } from '@/src/utils/backup-utils';
711
import { Parameters } from '@/src/utils/constants';
12+
import { findCognitoUser, getUserGroup } from './utils/cognito-utils';
13+
import {
14+
backupObject,
15+
copyObjects,
16+
deleteObjects,
17+
getItemObjects,
18+
} from './utils/s3-utils';
819

920
function getParameters(): Parameters {
1021
return yargs(hideBin(process.argv))
1122
.options({
12-
sourceOwner: {
13-
type: 'string',
14-
demandOption: true,
15-
},
16-
destinationOwner: {
17-
type: 'string',
18-
demandOption: true,
19-
},
2023
environment: {
2124
type: 'string',
2225
demandOption: true,
@@ -29,18 +32,73 @@ async function updateItems(
2932
items: Record<string, AttributeValue>[],
3033
parameters: Parameters
3134
): Promise<void> {
32-
for (let i = 0; i < items.length; i++) {
33-
const item = items[i];
34-
await updateItem(item, parameters);
35-
console.log(
36-
`Updated ${item.id.S}: ${i + 1}/${items.length} (${(100.0 * (i + 1)) / items.length}%)`
37-
);
35+
for (const item of items) {
36+
console.log(item.owner.S);
37+
38+
// Get owner id of this item
39+
const { owner, id, templateType, clientId } = item;
40+
41+
//check if owner doesn't have CLIENT#
42+
if (owner.S && !owner.S?.includes('CLIENT#')) {
43+
// check the user in cognito, if it exist then pull the client id
44+
const cognitoUser = await findCognitoUser(owner.S);
45+
46+
// check and get user groups - this is used when migrating for production
47+
const userClientIdFromGroup = await getUserGroup({
48+
Username: cognitoUser?.username,
49+
UserPoolId: cognitoUser?.poolId,
50+
});
51+
52+
// resolve client id
53+
const newClientId = (userClientIdFromGroup ??
54+
cognitoUser?.clientIdAttr) as string;
55+
56+
// check if this item has a client id, if yes check it matches the client id above. If it doesn't, throw error!
57+
if (item.clientId && item.clientId.S !== newClientId) {
58+
console.log({
59+
templateClientId: item.clientId,
60+
cognitoClientId: newClientId,
61+
});
62+
63+
throw new Error('Invalid ids');
64+
}
65+
// if it matches make the required swaps (clientId, createdby and updatedby)
66+
// if item doesn't have a client id then create one and do the above
67+
// update the item and delete the previous one
68+
await Promise.all([
69+
updateItem(item, parameters, newClientId),
70+
deleteItem(item, parameters),
71+
]);
72+
}
73+
74+
//
75+
if (templateType.S === 'LETTER') {
76+
//Retrieve item S3 object(s)
77+
const itemObjects = await getItemObjects(id.S as string);
78+
79+
//migrate to a new s3 location
80+
for (const itemObject of itemObjects) {
81+
const versionId = itemObject['Key'].split('/').reverse();
82+
await Promise.all([
83+
copyObjects(
84+
owner.S as string,
85+
itemObject['Key'],
86+
id.S as string,
87+
versionId[0],
88+
clientId.S as string
89+
),
90+
// delete previous objects
91+
deleteObjects(itemObject['Key']),
92+
]).then(() => console.log('Object moved'));
93+
}
94+
}
3895
}
3996
}
4097

4198
export async function performTransfer() {
4299
const parameters = getParameters();
43-
const items = await retrieveTemplates(parameters);
100+
const items = await retrieveAllTemplates(parameters);
44101
await backupData(items, parameters);
102+
await backupObject(parameters);
45103
await updateItems(items, parameters);
46104
}

‎data-migration/user-transfer/src/utils/backup-utils.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
/* eslint-disable security/detect-non-literal-fs-filename */
2+
import fs from 'node:fs';
23
import { AttributeValue } from '@aws-sdk/client-dynamodb';
34
import { Parameters } from '@/src/utils/constants';
45
import { getAccountId } from '@/src/utils/sts-utils';
@@ -13,12 +14,36 @@ export async function backupData(
1314
return;
1415
}
1516

16-
const { environment, sourceOwner, destinationOwner } = parameters;
17+
const { environment } = parameters;
1718
const accountId = await getAccountId();
1819
const bucketName = `nhs-notify-${accountId}-eu-west-2-main-acct-migration-backup`;
1920

2021
const timestamp = new Date().toISOString().replaceAll(/[.:T-]/g, '_');
21-
const filePath = `user-transfer/${environment}/${timestamp}-source-${sourceOwner}-destination-${destinationOwner}.json`;
22+
const filePath = `ownership-transfer/templates/templates-list/${environment}/${timestamp}.json`;
2223
await writeJsonToFile(filePath, JSON.stringify(items), bucketName);
2324
console.log(`Backed up data to s3://${bucketName}/${filePath}`);
2425
}
26+
27+
export async function backupDataLocal(
28+
items: Record<string, AttributeValue>[]
29+
): Promise<void> {
30+
console.log(`Found ${items.length} results`);
31+
if (items.length <= 0) {
32+
return;
33+
}
34+
35+
const timestamp = new Date().toISOString().replaceAll(/[.:T-]/g, '_');
36+
37+
fs.writeFile(
38+
`new-data-${timestamp}-templates.json`,
39+
JSON.stringify(items),
40+
(err) => {
41+
if (err) {
42+
console.log('Error writing file:', err);
43+
} else {
44+
console.log('Successfully wrote file');
45+
}
46+
}
47+
);
48+
console.log(`Data downloaded`);
49+
}

‎data-migration/user-transfer/src/utils/cognito-utils.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import {
2+
CognitoIdentityProviderClient,
3+
ListUsersCommand,
4+
AdminListGroupsForUserCommand,
5+
AdminListGroupsForUserCommandInput,
6+
UserType,
7+
GroupType,
8+
} from '@aws-sdk/client-cognito-identity-provider';
9+
10+
const cognito = new CognitoIdentityProviderClient({
11+
region: 'eu-west-2',
12+
});
13+
const USER_POOL_ID = 'eu-west-2_OX5mAA0VI';
14+
15+
interface CognitoUserBasics {
16+
username: string;
17+
sub?: string;
18+
clientIdAttr?: string;
19+
poolId: string;
20+
user?: UserType;
21+
}
22+
23+
export async function findCognitoUser(
24+
ownerId: string
25+
): Promise<CognitoUserBasics | undefined> {
26+
console.log('owner', ownerId);
27+
const listUser = new ListUsersCommand({
28+
UserPoolId: USER_POOL_ID,
29+
Filter: `"sub"="${ownerId}"`,
30+
});
31+
const res = await cognito.send(listUser);
32+
const user = res.Users?.[0];
33+
if (user) {
34+
const sub = user.Attributes?.find((a) => a.Name === 'sub')?.Value;
35+
const clientIdAttr = user.Attributes?.find(
36+
(a) => a.Name === 'custom:sbx_client_id' // this would be removed when migrating for production
37+
)?.Value;
38+
return {
39+
username: user.Username!,
40+
sub,
41+
clientIdAttr,
42+
poolId: USER_POOL_ID,
43+
user,
44+
};
45+
}
46+
return undefined;
47+
}
48+
49+
export async function getUserGroup(
50+
input: AdminListGroupsForUserCommandInput
51+
): Promise<string | undefined> {
52+
const { Username, UserPoolId } = input;
53+
54+
const userGroups = await cognito.send(
55+
new AdminListGroupsForUserCommand({
56+
UserPoolId,
57+
Username,
58+
})
59+
);
60+
61+
return userGroups.Groups && userGroups.Groups?.length === 0
62+
? undefined
63+
: await getUserClientId(userGroups.Groups);
64+
}
65+
66+
async function getUserClientId(
67+
userGroups: GroupType[] | undefined
68+
): Promise<string | undefined> {
69+
if (userGroups && userGroups.length > 0) {
70+
const clientIdGroup = userGroups?.filter((group) =>
71+
group.GroupName?.includes('client')
72+
);
73+
74+
return clientIdGroup[0].GroupName?.split(':')[0];
75+
}
76+
77+
return undefined;
78+
}

‎data-migration/user-transfer/src/utils/constants.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
export type Parameters = {
2-
sourceOwner: string;
3-
destinationOwner: string;
2+
sourceOwner?: string;
3+
destinationOwner?: string;
44
environment: string;
55
};

‎data-migration/user-transfer/src/utils/ddb-utils.ts‎

Lines changed: 56 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,42 @@
11
import {
22
AttributeValue,
3+
DeleteItemCommand,
34
DynamoDBClient,
5+
PutItemCommand,
46
QueryCommand,
57
QueryCommandInput,
8+
ScanCommand,
9+
ScanCommandInput,
610
TransactWriteItemsCommand,
711
} from '@aws-sdk/client-dynamodb';
812
import { Parameters } from '@/src/utils/constants';
913

1014
const client = new DynamoDBClient({ region: 'eu-west-2' });
1115

16+
// change 'sandbox' back to 'app' when testing on prod environment
1217
function getTableName(environment: string) {
13-
return `nhs-notify-${environment}-app-api-templates`;
18+
return `nhs-notify-${environment}-sandbox-api-templates`;
19+
}
20+
21+
export async function retrieveAllTemplates(
22+
parameters: Parameters
23+
): Promise<Record<string, AttributeValue>[]> {
24+
let allItems: Record<string, AttributeValue>[] = [];
25+
let lastEvaluatedKey = undefined;
26+
do {
27+
const query: ScanCommandInput = {
28+
TableName: getTableName(parameters.environment),
29+
FilterExpression:
30+
'attribute_exists(#owner) AND NOT contains(#owner, :subString)',
31+
ExpressionAttributeNames: { '#owner': 'owner' },
32+
ExpressionAttributeValues: { ':subString': { S: 'CLIENT#' } },
33+
};
34+
35+
const queryResult = await client.send(new ScanCommand(query));
36+
lastEvaluatedKey = queryResult.LastEvaluatedKey;
37+
allItems = [...allItems, ...(queryResult.Items ?? [])];
38+
} while (lastEvaluatedKey);
39+
return allItems;
1440
}
1541

1642
export async function retrieveTemplates(
@@ -26,7 +52,7 @@ export async function retrieveTemplates(
2652
'#owner': 'owner',
2753
},
2854
ExpressionAttributeValues: {
29-
':owner': { S: parameters.sourceOwner },
55+
':owner': { S: parameters.sourceOwner as string },
3056
},
3157
ExclusiveStartKey: lastEvaluatedKey,
3258
};
@@ -40,32 +66,37 @@ export async function retrieveTemplates(
4066

4167
export async function updateItem(
4268
item: Record<string, AttributeValue>,
43-
parameters: Parameters
69+
parameters: Parameters,
70+
newClientId: string
4471
): Promise<void> {
72+
const tableName = getTableName(parameters.environment);
73+
console.log({ item, parameters, newClientId, tableName });
74+
await client.send(
75+
new PutItemCommand({
76+
Item: {
77+
...item,
78+
owner: { S: `CLIENT#${newClientId}` },
79+
clientId: { S: newClientId },
80+
createdBy: item.owner,
81+
updatedBy: item.owner,
82+
},
83+
TableName: tableName,
84+
})
85+
);
86+
}
87+
88+
export async function deleteItem(
89+
item: Record<string, AttributeValue>,
90+
parameters: Parameters
91+
) {
4592
const tableName = getTableName(parameters.environment);
4693
await client.send(
47-
new TransactWriteItemsCommand({
48-
TransactItems: [
49-
{
50-
Put: {
51-
Item: {
52-
...item,
53-
owner: { S: parameters.destinationOwner },
54-
updatedAt: { S: new Date().toISOString() },
55-
},
56-
TableName: tableName,
57-
},
58-
},
59-
{
60-
Delete: {
61-
Key: {
62-
owner: item.owner,
63-
id: item.id,
64-
},
65-
TableName: tableName,
66-
},
67-
},
68-
],
94+
new DeleteItemCommand({
95+
Key: {
96+
owner: item.owner,
97+
},
98+
TableName: tableName,
6999
})
70100
);
101+
console.log('Item deleted');
71102
}

0 commit comments

Comments
 (0)