Merge pull request #20 from which-ecosystem/s3-reuploads

Improve reuploads and add script

Author: eug-vs

.eslintignore

@@ -0,0 +1 @@
+scripts

hooks/fetchImages.ts

@@ -1,7 +1,9 @@
 import { HookContext } from '@feathersjs/feathers';
 import Bluebird from 'bluebird';
 import _ from 'lodash';
+import Debug from 'debug';
 
+const debug = Debug('s3-reuploads');
 
 export default (paths: string[]) => async (context: HookContext): Promise<HookContext> => {
   const {
@@ -19,10 +21,13 @@ export default (paths: string[]) => async (context: HookContext): Promise<HookCo
 
     // If image is not from our s3, fetch it!
     if (!fileService.isS3url(url)) {
+      debug('Found non-s3 url!');
       const filePath = await fileService.downloadFile(url);
       const s3Path = fileService.generateS3Path(user?.username);
       const s3Url = await fileService.uploadFileToS3(filePath, s3Path);
-      return model.findOneAndUpdate({ _id: result._id }, { [path]: s3Url });
+      await model.findOneAndUpdate({ _id: result._id }, { $set: { [path]: s3Url } });
+      debug(`Fetched and updated: from ${url} to ${s3Url}`);
+      return s3Url;
     }
     return url;
   });

package-lock.json

@@ -23,6 +23,7 @@
     "@types/axios": "^0.14.0",
     "@types/bluebird": "^3.5.32",
     "@types/cors": "^2.8.6",
+    "@types/debug": "^4.1.5",
     "@types/lodash": "^4.14.155",
     "@types/mongoose": "^5.7.23",
     "@types/node": "^14.0.27",
@@ -33,6 +34,7 @@
     "axios": "^0.19.2",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
+    "debug": "^4.1.1",
     "feathers-hooks-common": "^5.0.3",
     "feathers-mongoose": "^8.3.0",
     "lodash": "^4.17.15",

package.json

@@ -0,0 +1,46 @@
+import mongoose from 'mongoose';
+import bluebird from 'bluebird';
+import _ from 'lodash';
+import {
+  User,
+  Poll,
+  Vote,
+  Feedback
+} from 'which-types';
+
+import app from '../app';
+app.service('files').setup(app);
+
+const MONGODB_URL = process.env.MONGODB_URI || 'mongodb://localhost:27017/which';
+
+mongoose.connect(MONGODB_URL, {
+  useNewUrlParser: true,
+  useUnifiedTopology: true,
+  useCreateIndex: true,
+  useFindAndModify: false,
+  family: 4 // Use IPv4, skip trying IPv6
+});
+
+const patchPoll = (poll: Poll): Promise<Poll> => {
+  console.log(`Patching poll of user ${poll.author.username}`)
+  return app.service('polls').patch(poll._id.toString(), {}, { user: poll.author, authenticated: true });
+};
+
+const patchUser = (user: User): Promise<User> => {
+  console.log(`Patching user ${user.username}`)
+  return app.service('users').patch(user._id.toString(), {}, { user, authenticated: true });
+};
+
+const update = async () => {
+  const users = app.service('users').find();
+
+  await bluebird.mapSeries(users, async (user: User) => {
+    await patchUser(user);
+    const polls = await app.service('polls').find({ query: { authorId: user._id }});
+    await bluebird.mapSeries(polls, (poll: Poll) => patchPoll(poll));
+    return;
+  });
+};
+
+update();
+

scripts/migrateImages.ts

@@ -8,7 +8,8 @@ import {
   Feedback
 } from 'which-types';
 
-import app from './app';
+import app from '../app';
+app.service('files').setup(app);
 
 const MONGODB_URL = process.env.MONGODB_URI || 'mongodb://localhost:27017/which';
 
@@ -20,7 +21,7 @@ mongoose.connect(MONGODB_URL, {
   family: 4 // Use IPv4, skip trying IPv6
 });
 
-const POLLS_AMOUNT = 20;
+const POLLS_AMOUNT = 5;
 
 const imageUrls: string[] = [
   // eslint-disable max-len
@@ -31,14 +32,14 @@ const imageUrls: string[] = [
 ];
 
 const names: string[] = [
-  'Emma',
-  'Elise',
-  'Jack',
-  'Oliver',
-  'Jamie',
-  'Adam',
-  'Jordan',
-  'William'
+  'emma',
+  'elise',
+  'jack',
+  'oliver',
+  'jamie',
+  'adam',
+  'jordan',
+  'william'
 ];
 
 const choices = [
@@ -101,5 +102,5 @@ const populate = async () => {
   });
 };
 
-populate().finally(mongoose.disconnect);
+populate();
 

populateDb.ts renamed to scripts/populateDb.ts

@@ -55,7 +55,8 @@ export default {
   },
   after: {
     all: convertPoll,
-    create: fetchImages(['contents.left.url', 'contents.right.url'])
+    create: fetchImages(['contents.left.url', 'contents.right.url']),
+    patch: fetchImages(['contents.left.url', 'contents.right.url'])
   }
 };