Skip to content

Commit 2b3be3a

Browse files
catarakcatarak
committed
[#1317] Update and run email consolidation script
1 parent cd5c000 commit 2b3be3a

File tree

6 files changed

+222
-95
lines changed

6 files changed

+222
-95
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ localhost.key
1919
privkey.pem
2020

2121
storybook-static
22+
duplicates.json

‎server/controllers/__mocks__/aws.controller.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
export const getObjectKey = jest.mock();
22
export const deleteObjectsFromS3 = jest.fn();
33
export const signS3 = jest.fn();
4-
export const copyObjectInS3 = jest.fn();
4+
export const copyObjectInS3RequestHandler = jest.fn();
55
export const listObjectsInS3ForUser = jest.fn();

‎server/controllers/aws.controller.js

Lines changed: 54 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -98,24 +98,41 @@ export function signS3(req, res) {
9898
res.json(result);
9999
}
100100

101-
export function copyObjectInS3(req, res) {
102-
const { url } = req.body;
103-
const objectKey = getObjectKey(url);
104-
const fileExtension = getExtension(objectKey);
105-
const newFilename = uuid.v4() + fileExtension;
106-
const userId = req.user.id;
107-
const params = {
108-
Bucket: `${process.env.S3_BUCKET}`,
109-
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
110-
Key: `${userId}/${newFilename}`,
111-
ACL: 'public-read'
112-
};
113-
const copy = client.copyObject(params);
114-
copy.on('err', (err) => {
115-
console.log(err);
101+
export function copyObjectInS3(url, userId) {
102+
return new Promise((resolve, reject) => {
103+
const objectKey = getObjectKey(url);
104+
const fileExtension = getExtension(objectKey);
105+
const newFilename = uuid.v4() + fileExtension;
106+
const headParams = {
107+
Bucket: `${process.env.S3_BUCKET}`,
108+
Key: `${objectKey}`
109+
};
110+
client.s3.headObject(headParams, (headErr) => {
111+
if (headErr) {
112+
reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
113+
return;
114+
}
115+
const params = {
116+
Bucket: `${process.env.S3_BUCKET}`,
117+
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
118+
Key: `${userId}/${newFilename}`,
119+
ACL: 'public-read'
120+
};
121+
const copy = client.copyObject(params);
122+
copy.on('err', (err) => {
123+
reject(err);
124+
});
125+
copy.on('end', (data) => {
126+
resolve(`${s3Bucket}${userId}/${newFilename}`);
127+
});
128+
});
116129
});
117-
copy.on('end', (data) => {
118-
res.json({ url: `${s3Bucket}${userId}/${newFilename}` });
130+
}
131+
132+
export function copyObjectInS3RequestHandler(req, res) {
133+
const { url } = req.body;
134+
copyObjectInS3(url, req.user.id).then((newUrl) => {
135+
res.json({ url: newUrl });
119136
});
120137
}
121138

@@ -124,19 +141,28 @@ export function moveObjectToUserInS3(url, userId) {
124141
const objectKey = getObjectKey(url);
125142
const fileExtension = getExtension(objectKey);
126143
const newFilename = uuid.v4() + fileExtension;
127-
console.log(`${process.env.S3_BUCKET}/${objectKey}`);
128-
const params = {
144+
const headParams = {
129145
Bucket: `${process.env.S3_BUCKET}`,
130-
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
131-
Key: `${userId}/${newFilename}`,
132-
ACL: 'public-read'
146+
Key: `${objectKey}`
133147
};
134-
const move = client.moveObject(params);
135-
move.on('err', (err) => {
136-
reject(err);
137-
});
138-
move.on('end', (data) => {
139-
resolve(`${s3Bucket}${userId}/${newFilename}`);
148+
client.s3.headObject(headParams, (headErr) => {
149+
if (headErr) {
150+
reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
151+
return;
152+
}
153+
const params = {
154+
Bucket: `${process.env.S3_BUCKET}`,
155+
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
156+
Key: `${userId}/${newFilename}`,
157+
ACL: 'public-read'
158+
};
159+
const move = client.moveObject(params);
160+
move.on('err', (err) => {
161+
reject(err);
162+
});
163+
move.on('end', (data) => {
164+
resolve(`${s3Bucket}${userId}/${newFilename}`);
165+
});
140166
});
141167
});
142168
}

‎server/migrations/emailConsolidation.js

Lines changed: 164 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import mongoose from 'mongoose';
2+
import fs from 'fs';
23
import User from '../models/user';
34
import Project from '../models/project';
45
import Collection from '../models/collection';
5-
import { moveObjectToUserInS3 } from '../controllers/aws.controller';
6+
import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller';
67
import mail from '../utils/mail';
78
import { renderAccountConsolidation } from '../views/mail';
89

@@ -65,73 +66,172 @@ const agg = [
6566
// then, send the email
6667
// then, figure out how to iterate through all of the users.
6768

69+
// create list of duplicate users
70+
// User.aggregate(agg).then((result) => {
71+
// return fs.writeFile('duplicates.json', JSON.stringify(result), () => {
72+
// console.log('File written.');
73+
// process.exit(0);
74+
// });
75+
// });
76+
6877
let currentUser = null;
6978
let duplicates = null;
70-
User.aggregate(agg).then((result) => {
71-
console.log(result);
72-
const email = result[0]._id;
79+
80+
fs.readFile('duplicates.json', async (err, file) => {
81+
const result = JSON.parse(file);
82+
for (let i = 3000; i < result.length; i += 1) {
83+
console.log('Index: ', i);
84+
const email = result[i]._id;
85+
console.log(email);
86+
await consolidateAccount(email); // eslint-disable-line
87+
}
88+
process.exit(0);
89+
});
90+
91+
async function consolidateAccount(email) {
7392
return User.find({ email }).collation({ locale: 'en', strength: 2 })
74-
.sort({ createdAt: 1 }).exec();
75-
}).then((result) => {
76-
[currentUser, ...duplicates] = result;
77-
console.log('Current User: ', currentUser._id, ' ', currentUser.email);
78-
duplicates = duplicates.map(dup => dup._id);
79-
console.log('Duplicates: ', duplicates);
80-
return Project.find({
81-
user: { $in: duplicates }
82-
}).exec();
83-
}).then((sketches) => {
84-
const saveSketchPromises = [];
85-
sketches.forEach((sketch) => {
86-
const moveSketchFilesPromises = [];
87-
sketch.files.forEach((file) => {
88-
if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) {
89-
const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
90-
.then((newUrl) => {
91-
file.url = newUrl;
92-
});
93-
moveSketchFilesPromises.push(fileSavePromise);
94-
}
95-
});
96-
const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
97-
sketch.user = ObjectId(currentUser._id);
98-
return sketch.save();
93+
.sort({ createdAt: 1 }).exec()
94+
.then((result) => {
95+
[currentUser, ...duplicates] = result;
96+
console.log('Current User: ', currentUser._id, ' ', currentUser.email);
97+
duplicates = duplicates.map(dup => dup._id);
98+
console.log('Duplicates: ', duplicates);
99+
return Project.find({
100+
user: { $in: duplicates }
101+
}).exec();
102+
}).then((sketches) => {
103+
const saveSketchPromises = [];
104+
sketches.forEach((sketch) => {
105+
console.log('SketchId: ', sketch._id);
106+
console.log('UserId: ', sketch.user);
107+
const moveSketchFilesPromises = [];
108+
sketch.files.forEach((file) => {
109+
// if the file url contains sketch user
110+
if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) {
111+
if (file.url.includes(sketch.user)) {
112+
const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
113+
.then((newUrl) => {
114+
file.url = newUrl;
115+
}).catch((err) => {
116+
console.log('Move Error:');
117+
console.log(err);
118+
});
119+
moveSketchFilesPromises.push(fileSavePromise);
120+
} else {
121+
const fileSavePromise = copyObjectInS3(file.url, currentUser._id)
122+
.then((newUrl) => {
123+
file.url = newUrl;
124+
}).catch((err) => {
125+
console.log('Copy Error:');
126+
console.log(err);
127+
});
128+
moveSketchFilesPromises.push(fileSavePromise);
129+
}
130+
}
131+
});
132+
const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
133+
sketch.user = ObjectId(currentUser._id);
134+
return sketch.save();
135+
});
136+
saveSketchPromises.push(sketchSavePromise);
137+
});
138+
return Promise.all(saveSketchPromises);
139+
}).then(() => {
140+
console.log('Moved and updated all sketches.');
141+
return Collection.updateMany(
142+
{ owner: { $in: duplicates } },
143+
{ $set: { owner: ObjectId(currentUser.id) } }
144+
);
145+
}).then(() => {
146+
console.log('Moved and updated all collections.');
147+
return User.deleteMany({ _id: { $in: duplicates } });
148+
}).then(() => {
149+
console.log('Deleted other user accounts.');
150+
currentUser.email = currentUser.email.toLowerCase();
151+
return currentUser.save();
152+
}).then(() => {
153+
console.log('Migrated email to lowercase.');
154+
// const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
155+
const mailOptions = renderAccountConsolidation({
156+
body: {
157+
domain: 'https://editor.p5js.org',
158+
username: currentUser.username,
159+
email: currentUser.email
160+
},
161+
to: currentUser.email,
162+
});
163+
164+
return new Promise((resolve, reject) => {
165+
mail.send(mailOptions, (mailErr, result) => {
166+
console.log('Sent email.');
167+
if (mailErr) {
168+
return reject(mailErr);
169+
}
170+
return resolve(result);
171+
});
172+
});
173+
}).catch((err) => {
174+
console.log(err);
175+
process.exit(1);
99176
});
100-
saveSketchPromises.push(sketchSavePromise);
101-
});
102-
return Promise.all(saveSketchPromises);
103-
}).then(() => {
104-
console.log('Moved and updated all sketches.');
105-
return Collection.updateMany(
106-
{ owner: { $in: duplicates } },
107-
{ $set: { owner: ObjectId(currentUser.id) } }
108-
);
109-
}).then(() => {
110-
console.log('Moved and updated all collections.');
111-
return User.deleteMany({ _id: { $in: duplicates } });
112-
}).then(() => {
113-
console.log('Deleted other user accounts.');
114-
currentUser.email = currentUser.email.toLowerCase();
115-
return currentUser.save();
116-
}).then(() => {
117-
console.log('Migrated email to lowercase.');
118-
// const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
119-
const mailOptions = renderAccountConsolidation({
120-
body: {
121-
domain: 'https://editor.p5js.org',
122-
username: currentUser.username,
123-
email: currentUser.email
124-
},
125-
to: currentUser.email,
126-
});
127-
128-
mail.send(mailOptions, (mailErr, result) => {
129-
console.log('Sent email.');
130-
process.exit(0);
131-
});
132-
});
177+
}
178+
179+
180+
// let duplicates = [
181+
// "5ce3d936e0f9df0022d8330c",
182+
// "5cff843f091745001e83c070",
183+
// "5d246f5db489e6001eaee6e9"
184+
// ];
185+
// let currentUser = null;
186+
// User.deleteMany({ _id: { $in: duplicates } }).then(() => {
187+
// return User.findOne({ "email": "Silverstar09@hotmail.com" })
188+
// }).then((result) => {
189+
// currentUser = result;
190+
// console.log('Deleted other user accounts.');
191+
// currentUser.email = currentUser.email.toLowerCase();
192+
// return currentUser.save();
193+
// }).then(() => {
194+
// const mailOptions = renderAccountConsolidation({
195+
// body: {
196+
// domain: 'https://editor.p5js.org',
197+
// username: currentUser.username,
198+
// email: currentUser.email
199+
// },
200+
// to: currentUser.email,
201+
// });
202+
203+
// return new Promise((resolve, reject) => {
204+
// mail.send(mailOptions, (mailErr, result) => {
205+
// console.log('Sent email.');
206+
// if (mailErr) {
207+
// return reject(mailErr);
208+
// }
209+
// return resolve(result);
210+
// });
211+
// });
212+
// });
213+
133214

134-
// ).then((result) => {
135-
// console.log(result);
215+
// import s3 from '@auth0/s3';
216+
217+
// const client = s3.createClient({
218+
// maxAsyncS3: 20,
219+
// s3RetryCount: 3,
220+
// s3RetryDelay: 1000,
221+
// multipartUploadThreshold: 20971520, // this is the default (20 MB)
222+
// multipartUploadSize: 15728640, // this is the default (15 MB)
223+
// s3Options: {
224+
// accessKeyId: `${process.env.AWS_ACCESS_KEY}`,
225+
// secretAccessKey: `${process.env.AWS_SECRET_KEY}`,
226+
// region: `${process.env.AWS_REGION}`
227+
// },
136228
// });
137229

230+
// const headParams = {
231+
// Bucket: `${process.env.S3_BUCKET}`,
232+
// Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png"
233+
// };
234+
// client.s3.headObject(headParams, (err, data) => {
235+
// console.log(err);
236+
// console.log(data);
237+
// });

‎server/migrations/start.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
require('@babel/register');
22
require('@babel/polyfill');
33
const path = require('path');
4-
require('dotenv').config({ path: path.resolve('.env') });
4+
require('dotenv').config({ path: path.resolve('.env.production') });
55
require('./emailConsolidation');
66
// require('./populateTotalSize');
77
// require('./moveBucket');

‎server/routes/aws.routes.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated';
55
const router = new Router();
66

77
router.post('/S3/sign', isAuthenticated, AWSController.signS3);
8-
router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3);
8+
router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler);
99
router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3);
1010
router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);
1111

0 commit comments

Comments
 (0)