Skip to content

Commit fea3e97

Browse files
bhansell1chris-elliott-nhsd
bhansell1
authored and
chris-elliott-nhsd
committed
CCM-10429: use backedup data as source for migration
1 parent de9ae61 commit fea3e97

File tree

5 files changed

+116
-82
lines changed

5 files changed

+116
-82
lines changed

‎data-migration/user-transfer/src/migrate.ts‎

Lines changed: 78 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
/* eslint-disable security/detect-non-literal-fs-filename */
2-
/* eslint-disable unicorn/prefer-top-level-await */
31
import fs from 'node:fs';
42
import path from 'node:path';
53
import yargs from 'yargs';
@@ -8,8 +6,12 @@ import { UserTransferPlan, UserTransferPlanItem } from './utils/types';
86
import { print } from './utils/log-utils';
97
import { UserTransfer } from './utils/user-transfer';
108
import { getTemplates } from './utils/ddb-utils';
11-
import { backupBucketName, backupData } from './utils/backup-utils';
12-
import { transferFileToNewBucket, writeFile } from './utils/s3-utils';
9+
import { backupBucketName, backupData, writeLocal } from './utils/backup-utils';
10+
import {
11+
transferFileToNewBucket,
12+
writeFile as writeRemote,
13+
} from './utils/s3-utils';
14+
import { AttributeValue } from '@aws-sdk/client-dynamodb';
1315

1416
const params = yargs(hideBin(process.argv))
1517
.options({
@@ -28,41 +30,30 @@ const params = yargs(hideBin(process.argv))
2830
})
2931
.parseSync();
3032

31-
function writeLocal(filename: string, data: string) {
32-
fs.writeFile(filename, data, (err) => {
33-
if (err) {
34-
console.log(`Error writing file: ${filename}`, err);
35-
} else {
36-
console.log(`Successfully wrote ${filename}`);
37-
}
38-
});
39-
}
40-
41-
async function backup(
33+
async function loadTemplates(
4234
tableName: string,
43-
sourceBucket: string,
44-
backupBucket: string,
4535
migrations: UserTransferPlanItem[]
4636
) {
4737
const keys = migrations.map((r) => ({
4838
id: r.templateId,
4939
owner: r.ddb.owner.from,
5040
}));
5141

52-
const files = migrations.flatMap((r) => r.s3.files.flatMap((a) => a.from));
53-
54-
const data = await getTemplates(tableName, keys);
42+
return await getTemplates(tableName, keys);
43+
}
5544

56-
if (data.length !== migrations.length) {
57-
throw new Error(
58-
`Only retrieved ${data.length}/${migrations.length} migrations`
59-
);
60-
}
45+
async function backup(
46+
templates: Record<string, AttributeValue>[],
47+
sourceBucket: string,
48+
backupBucket: string,
49+
migrations: UserTransferPlanItem[]
50+
) {
51+
const files = migrations.flatMap((r) => r.s3.files.flatMap((a) => a.from));
6152

6253
const { name } = path.parse(params.file);
6354

6455
await backupData(
65-
data,
56+
templates,
6657
backupBucket,
6758
`ownership-transfer/${params.environment}/${name}`
6859
);
@@ -79,10 +70,33 @@ async function backup(
7970
);
8071
}
8172

73+
function assertMatchingTemplates(
74+
fetchedTemplateIds: string[],
75+
migrationTemplateIds: string[]
76+
) {
77+
const left = new Set(fetchedTemplateIds);
78+
const right = new Set(migrationTemplateIds);
79+
80+
if (left.size !== right.size) {
81+
throw new Error(
82+
`Mismatch in length of fetched templates and templates to migrate`
83+
);
84+
}
85+
86+
for (const value of left) {
87+
if (!right.has(value)) {
88+
throw new Error(
89+
`Value "${value}" found in first array but not in second`
90+
);
91+
}
92+
}
93+
}
94+
8295
async function migrate() {
8396
const backupBucket = await backupBucketName();
8497

8598
const input = JSON.parse(
99+
// eslint-disable-next-line security/detect-non-literal-fs-filename
86100
fs.readFileSync(params.file, 'utf8')
87101
) as UserTransferPlan;
88102

@@ -94,26 +108,47 @@ async function migrate() {
94108

95109
print(`Total migrations: ${migrations.length}`);
96110

111+
const templates = await loadTemplates(input.tableName, migrations);
112+
113+
assertMatchingTemplates(
114+
templates.map((r) => r.id.S!),
115+
migrations.map((r) => r.templateId)
116+
);
117+
97118
if (!params.dryRun) {
98-
await backup(input.tableName, input.bucketName, backupBucket, migrations);
119+
await backup(templates, input.bucketName, backupBucket, migrations);
120+
print('Data backed up');
99121
}
100122

101-
for (let i = 0; i < migrations.length; i++) {
102-
const migration = migrations[i];
123+
let count = 0;
124+
125+
for (const migration of migrations) {
126+
count += 1;
127+
128+
print(`Progress: ${count}/${migrations.length}`);
129+
print(`Processing: ${migration.templateId}`);
130+
131+
const template = templates.find((r) => r.id.S === migration.templateId);
132+
133+
if (!template) {
134+
print(
135+
`Skipping: Unable to find template ${migration.templateId} in backup data`
136+
);
137+
continue;
138+
}
103139

104140
const idx = output.migrate.plans.findIndex(
105141
(r) => r.templateId === migration.templateId
106142
);
107143

108144
if (idx === -1) {
109-
print(`Skipping: Unable to locate index for ${migration.templateId}`);
145+
print(
146+
`Skipping: Unable to locate index in output data for ${migration.templateId}`
147+
);
110148
continue;
111149
}
112150

113-
print(`Progress: ${i + 1}/${migrations.length}`);
114-
print(`Processing: ${migration.templateId}`);
115-
116-
const result = await UserTransfer.apply(migration, {
151+
const result = await UserTransfer.apply(migration, template, {
117152
bucketName: input.bucketName,
118153
tableName: input.tableName,
119154
dryRun: params.dryRun,
@@ -130,20 +165,21 @@ async function migrate() {
130165
}
131166

132167
const { dir, name, ext } = path.parse(params.file);
133-
134-
const fileName = `${name}-${params.dryRun ? 'dryrun' : ''}${ext}`;
135-
168+
const filename = `${name}-${params.dryRun ? 'dryrun' : 'run'}${ext}`;
136169
const data = JSON.stringify(output);
137170

138-
const filePath = `ownership-transfer/${params.environment}/${name}/${fileName}`;
139-
140-
writeLocal(path.join(dir, fileName), data);
171+
writeLocal(path.join(dir, filename), data);
141172

142-
await writeFile(filePath, data, backupBucket);
173+
await writeRemote(
174+
`ownership-transfer/${params.environment}/${name}/${filename}`,
175+
data,
176+
backupBucket
177+
);
143178

144-
print(`Plan written to s3://${backupBucket}/${filePath}`);
179+
print(`Results written to ${filename}`);
145180
}
146181

147182
migrate()
148183
.then(() => console.log('finished'))
184+
// eslint-disable-next-line unicorn/prefer-top-level-await
149185
.catch((error) => console.error(error));

‎data-migration/user-transfer/src/plan.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
1-
/* eslint-disable unicorn/no-process-exit */
2-
/* eslint-disable unicorn/prefer-top-level-await */
31
import { CognitoRepository } from './utils/cognito-repository';
42
import { listAllTemplates } from './utils/ddb-utils';
5-
import { listAllFiles, writeFile } from './utils/s3-utils';
3+
import { listAllFiles, writeFile as writeRemote } from './utils/s3-utils';
64
import { UserTransfer } from './utils/user-transfer';
75
import yargs from 'yargs';
86
import { hideBin } from 'yargs/helpers';
97
import { getAccountId } from './utils/sts-utils';
108
import { print } from './utils/log-utils';
11-
import { backupBucketName } from './utils/backup-utils';
9+
import { backupBucketName, writeLocal } from './utils/backup-utils';
1210

1311
const params = yargs(hideBin(process.argv))
1412
.options({
@@ -99,18 +97,20 @@ async function plan() {
9997
}
10098
);
10199

102-
const fileName = `transfer-plan-${accountId}-${params.environment}-${params.component}-${Date.now()}`;
103-
100+
const filename = `transfer-plan-${accountId}-${params.environment}-${params.component}-${Date.now()}`;
104101
const data = JSON.stringify(transferPlan);
102+
const path = `ownership-transfer/${params.environment}/${filename}/${filename}.json`;
105103

106-
const filePath = `ownership-transfer/${params.environment}/${fileName}/${fileName}.json`;
104+
await writeRemote(path, data, templatesS3BackupBucketName);
107105

108-
await writeFile(filePath, data, templatesS3BackupBucketName);
106+
writeLocal(`./migrations/${filename}.json`, data);
109107

110-
print(`Plan written to s3://${templatesS3InternalBucketName}/${filePath}`);
108+
print(`Results written to ${filename}.json`);
111109
}
112110

111+
// eslint-disable-next-line unicorn/prefer-top-level-await
113112
plan().catch((error) => {
114113
console.error(error);
114+
// eslint-disable-next-line unicorn/no-process-exit
115115
process.exit(1);
116116
});

‎data-migration/user-transfer/src/utils/backup-utils.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { AttributeValue } from '@aws-sdk/client-dynamodb';
22
import { getAccountId } from './sts-utils';
33
import { writeFile } from './s3-utils';
4+
import fs from 'node:fs';
45

56
export const backupBucketName = async () => {
67
const accountId = await getAccountId();
@@ -22,3 +23,14 @@ export async function backupData(
2223
await writeFile(filePath, JSON.stringify(items), bucketName);
2324
console.log(`Backed up data to s3://${bucketName}/${filePath}`);
2425
}
26+
27+
export function writeLocal(filename: string, data: string) {
28+
// eslint-disable-next-line security/detect-non-literal-fs-filename
29+
fs.writeFile(filename, data, (err: unknown) => {
30+
if (err) {
31+
console.log(`Error writing file: ${filename}`, err);
32+
} else {
33+
console.log(`Successfully wrote ${filename}`);
34+
}
35+
});
36+
}

‎data-migration/user-transfer/src/utils/ddb-utils.ts‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,16 +109,10 @@ export async function getTemplates(
109109

110110
export async function migrateOwnership(
111111
tableName: string,
112-
templateId: string,
112+
template: Record<string, AttributeValue>,
113113
from: string,
114114
to: string
115115
) {
116-
const template = await getTemplate(tableName, from, templateId);
117-
118-
if (!template) {
119-
throw new Error(`No template found for ${templateId}`);
120-
}
121-
122116
const cmd = new TransactWriteItemsCommand({
123117
TransactItems: [
124118
{

‎data-migration/user-transfer/src/utils/user-transfer.ts‎

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ import {
55
Template,
66
UserData,
77
} from './types';
8-
import { getTemplate, migrateOwnership } from './ddb-utils';
8+
import { migrateOwnership } from './ddb-utils';
99
import { print } from './log-utils';
1010
import { transferFileToClient, deleteFile, getFileHead } from './s3-utils';
11+
import { AttributeValue } from '@aws-sdk/client-dynamodb';
1112

1213
export class UserTransfer {
1314
public static async plan(
@@ -56,7 +57,7 @@ export class UserTransfer {
5657
bucketName: s3.bucketName,
5758
migrate: {
5859
count: templatesToMigrate.length,
59-
plans: templatesToMigrate,
60+
plans: templatesToMigrate.filter((r) => r.s3.files.length > 0),
6061
},
6162
orphaned: {
6263
count: templateIdsWithNoOwner.length,
@@ -66,10 +67,11 @@ export class UserTransfer {
6667
}
6768

6869
public static async apply(
69-
item: UserTransferPlanItem,
70+
migration: UserTransferPlanItem,
71+
template: Record<string, AttributeValue>,
7072
config: { bucketName: string; tableName: string; dryRun: boolean }
7173
): Promise<UserTransferPlanItemResult> {
72-
const copyPromises = item.s3.files.map(async (file) => {
74+
const copyPromises = migration.s3.files.map(async (file) => {
7375
if (config.dryRun) {
7476
await getFileHead(config.bucketName, file.from);
7577
print(`[DRY RUN] S3: transfer ${file.from} to ${file.to}`);
@@ -78,15 +80,15 @@ export class UserTransfer {
7880
config.bucketName,
7981
file.from,
8082
file.to,
81-
item.ddb.owner.to
83+
migration.ddb.owner.to
8284
);
8385
}
8486
});
8587

8688
const copyResult = await UserTransfer.processPromises(copyPromises);
8789

8890
if (!copyResult.success) {
89-
print(`Skipping: [s3:copy]: ${item.templateId} copy failed`);
91+
print(`Skipping: [s3:copy]: ${migration.templateId} copy failed`);
9092

9193
return {
9294
...copyResult,
@@ -96,33 +98,23 @@ export class UserTransfer {
9698

9799
try {
98100
if (config.dryRun) {
99-
const template = await getTemplate(
100-
config.tableName,
101-
item.ddb.owner.from,
102-
item.templateId
103-
);
104-
105-
if (!template) {
106-
throw new Error(`No template found for ${item.templateId}`);
107-
}
108-
109101
print(
110-
`[DRY RUN] DynamoDB: template ${template.id.S} found and will transferred from ${item.ddb.owner.from} to CLIENT#${item.ddb.owner.to}`
102+
`[DRY RUN] DynamoDB: template ${template.id.S} found and will transferred from ${migration.ddb.owner.from} to CLIENT#${migration.ddb.owner.to}`
111103
);
112104
print(
113-
`[DRY RUN] DynamoDB: template ${template.id.S} with owner ${item.ddb.owner.from} will be deleted`
105+
`[DRY RUN] DynamoDB: template ${template.id.S} with owner ${migration.ddb.owner.from} will be deleted`
114106
);
115107
} else {
116108
await migrateOwnership(
117109
config.tableName,
118-
item.templateId,
119-
item.ddb.owner.from,
120-
item.ddb.owner.to
110+
template,
111+
migration.ddb.owner.from,
112+
migration.ddb.owner.to
121113
);
122114
}
123115
} catch (error) {
124116
print(
125-
`Failed: [ddb:transfer]: ${item.templateId} DynamoDB transaction failed`
117+
`Failed: [ddb:transfer]: ${migration.templateId} DynamoDB transaction failed`
126118
);
127119

128120
return {
@@ -135,7 +127,7 @@ export class UserTransfer {
135127
};
136128
}
137129

138-
const deletePromises = item.s3.files.map(async (file) => {
130+
const deletePromises = migration.s3.files.map(async (file) => {
139131
if (config.dryRun) {
140132
print(`[DRY RUN] S3: will delete ${file.from}`);
141133
} else {
@@ -146,7 +138,7 @@ export class UserTransfer {
146138
const deleteResult = await UserTransfer.processPromises(deletePromises);
147139

148140
if (!deleteResult.success) {
149-
print(`Partial: [s3:delete]: ${item.templateId} delete failed`);
141+
print(`Partial: [s3:delete]: ${migration.templateId} delete failed`);
150142

151143
return {
152144
...deleteResult,

0 commit comments

Comments
 (0)