Merge pull request #19 from which-ecosystem/s3-reuploads

S3 reuploads

Author: eug-vs

.gitignore

@@ -1,2 +1,4 @@
 /node_modules
-/.idea
+/.idea
+/tmp
+.env

hooks/fetchImages.ts

@@ -0,0 +1,31 @@
+import { HookContext } from '@feathersjs/feathers';
+import Bluebird from 'bluebird';
+import _ from 'lodash';
+
+
+export default (paths: string[]) => async (context: HookContext): Promise<HookContext> => {
+  const {
+    service,
+    app,
+    result,
+    params: { user }
+  } = context;
+
+  const fileService = app.service('files');
+  const model = service.Model;
+
+  Bluebird.map(paths, async (path: string) => {
+    const url = _.get(result, path);
+
+    // If image is not from our s3, fetch it!
+    if (!fileService.isS3url(url)) {
+      const filePath = await fileService.downloadFile(url);
+      const s3Path = fileService.generateS3Path(user?.username);
+      const s3Url = await fileService.uploadFileToS3(filePath, s3Path);
+      return model.findOneAndUpdate({ _id: result._id }, { [path]: s3Url });
+    }
+    return url;
+  });
+  return context;
+};
+

package-lock.json

@@ -20,6 +20,7 @@
     "@feathersjs/socketio": "^4.5.4",
     "@feathersjs/transport-commons": "^4.5.3",
     "@types/aws-sdk": "^2.6.1",
+    "@types/axios": "^0.14.0",
     "@types/bluebird": "^3.5.32",
     "@types/cors": "^2.8.6",
     "@types/lodash": "^4.14.155",
@@ -29,6 +30,7 @@
     "@typescript-eslint/eslint-plugin": "^3.2.0",
     "@typescript-eslint/parser": "^3.2.0",
     "aws-sdk": "^2.6.1",
+    "axios": "^0.19.2",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "feathers-hooks-common": "^5.0.3",

package.json

@@ -1,27 +1,86 @@
 import { Application } from '@feathersjs/express';
 import { Params } from '@feathersjs/feathers';
 import { v4 } from 'uuid';
+import axios from 'axios';
+import fs from 'fs';
 
 // Use require to avoid bug
 // https://stackoverflow.com/questions/62611373/heroku-crashes-when-importing-aws-sdk
+// TODO: use import statement
+// eslint-disable-next-line
 const S3 = require('aws-sdk/clients/s3');
 
 
 export default class Files {
-  app!: Application;
-  s3!: any;
-  bucket!: string;
+  public app!: Application;
+
+  private s3!: typeof S3;
+
+  private bucket!: string;
 
   async find(params: Params): Promise<string> {
+    const path = this.generateS3Path(params.user?.username);
+    return this.getUploadUrl(path);
+  }
+
+  public isS3url(url: string): boolean {
+    return url.startsWith(`https://${this.bucket}.s3`);
+  }
+
+  public generateS3Path(prefix = '', ext = 'png'): string {
+    const key = v4();
+    const fileName = `${key}.${ext}`;
+    return prefix ? `${prefix}/${fileName}` : fileName;
+  }
+
+  async getUploadUrl(path: string): Promise<string> {
     // Return signed upload URL
     return this.s3.getSignedUrl('putObject', {
       Bucket: this.bucket,
-      Key: `${params.user?.username}/${v4()}.png`,
+      Key: path,
       ContentType: 'image/*',
-      Expires: 300,
+      Expires: 300
     });
   }
 
+  async getDownloadUrl(path: string): Promise<string> {
+    return this.getUploadUrl(path).then((url: string) => {
+      const queryIndex = url.indexOf('?');
+      return url.slice(0, queryIndex);
+    });
+  }
+
+  private createTmpDir() {
+    if (!fs.existsSync('tmp')) fs.mkdirSync('tmp');
+  }
+
+  async downloadFile(url: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      this.createTmpDir();
+      const filePath = `tmp/${v4()}`;
+      const fileStream = fs.createWriteStream(filePath);
+      axios.get(url, { responseType: 'stream' })
+        .then(response => {
+          response.data.pipe(fileStream)
+            .on('error', reject)
+            .on('close', () => resolve(filePath));
+        })
+        .catch(error => reject(error));
+    });
+  }
+
+  async uploadFileToS3(filePath: string, s3Path: string): Promise<string> {
+    const fileStream = fs.createReadStream(filePath);
+    await this.s3.upload({
+      Bucket: this.bucket,
+      Key: s3Path,
+      Body: fileStream,
+      ContentType: 'image/png'
+    }).promise();
+    fs.unlinkSync(filePath);
+    return this.getDownloadUrl(s3Path);
+  }
+
   setup(app: Application): void {
     this.app = app;
     this.s3 = new S3({

services/files/files.class.ts

@@ -8,6 +8,7 @@ import { PollSchema } from '../../models/polls/poll.schema';
 import VoteModel from '../../models/votes/vote.model';
 import sortByDate from '../../hooks/sortByDate';
 import signAuthority from '../../hooks/signAuthority';
+import fetchImages from '../../hooks/fetchImages';
 
 
 const convertPoll = async (context: HookContext): Promise<HookContext> => {
@@ -53,7 +54,8 @@ export default {
     patch: disallow('external')
   },
   after: {
-    all: convertPoll
+    all: convertPoll,
+    create: fetchImages(['contents.left.url', 'contents.right.url'])
   }
 };
 

services/polls/polls.hooks.ts

@@ -4,6 +4,7 @@ import { discard, disallow } from 'feathers-hooks-common';
 import { HookContext } from '@feathersjs/feathers';
 import { NotAuthenticated } from '@feathersjs/errors';
 import requireAuth from '../../hooks/requireAuth';
+import fetchImages from '../../hooks/fetchImages';
 
 const hashPassword = hooks.hashPassword('password');
 
@@ -24,6 +25,8 @@ const compareUser = async (context: HookContext): Promise<HookContext> => {
 export default {
   after: {
     all: hooks.protect('password'),
+    create: fetchImages(['avatarUrl']),
+    patch: fetchImages(['avatarUrl']),
     get: discard('password') // Protect password from local get's
   },
   before: {