add script to add total size to all user accounts, add totalSize to user model, return totalSize from user api request

Author: catarak

server/controllers/aws.controller.js

@@ -108,54 +108,67 @@ export function copyObjectInS3(req, res) {
   });
 }
 
-export function listObjectsInS3ForUser(req, res) {
-  const { username } = req.user;
-  findUserByUsername(username, (user) => {
-    const userId = user.id;
+export function listObjectsInS3ForUser(userId) {
+  let assets = [];
+  return new Promise((resolve) => {
     const params = {
       s3Params: {
         Bucket: `${process.env.S3_BUCKET}`,
         Prefix: `${userId}/`
       }
     };
-    let assets = [];
     client.listObjects(params)
       .on('data', (data) => {
         assets = assets.concat(data.Contents.map(object => ({ key: object.Key, size: object.Size })));
       })
       .on('end', () => {
-        const projectAssets = [];
-        getProjectsForUserId(userId).then((projects) => {
-          let totalSize = 0;
-          assets.forEach((asset) => {
-            const name = asset.key.split('/').pop();
-            const foundAsset = {
-              key: asset.key,
-              name,
-              size: asset.size,
-              url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}`
-            };
-            totalSize += asset.size;
-            projects.some((project) => {
-              let found = false;
-              project.files.some((file) => {
-                if (!file.url) return false;
-                if (file.url.includes(asset.key)) {
-                  found = true;
-                  foundAsset.name = file.name;
-                  foundAsset.sketchName = project.name;
-                  foundAsset.sketchId = project.id;
-                  foundAsset.url = file.url;
-                  return true;
-                }
-                return false;
-              });
-              return found;
-            });
-            projectAssets.push(foundAsset);
+        resolve();
+      });
+    }).then(() => {
+      return getProjectsForUserId(userId);
+    }).then((projects) => {
+      const projectAssets = [];
+      let totalSize = 0;
+      assets.forEach((asset) => {
+        const name = asset.key.split('/').pop();
+        const foundAsset = {
+          key: asset.key,
+          name,
+          size: asset.size,
+          url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}`
+        };
+        totalSize += asset.size;
+        projects.some((project) => {
+          let found = false;
+          project.files.some((file) => {
+            if (!file.url) return false;
+            if (file.url.includes(asset.key)) {
+              found = true;
+              foundAsset.name = file.name;
+              foundAsset.sketchName = project.name;
+              foundAsset.sketchId = project.id;
+              foundAsset.url = file.url;
+              return true;
+            }
+            return false;
           });
-          res.json({ assets: projectAssets, totalSize });
+          return found;
         });
+        projectAssets.push(foundAsset);
       });
+      return Promise.resolve({ assets: projectAssets, totalSize});
+    }).catch((err) => {
+      console.log('got an error');
+      console.log(err);
+    });
+}
+
+export function listObjectsInS3ForUserRequestHandler(req, res) {
+  const { username } = req.user;
+  findUserByUsername(username, (user) => {
+    const userId = user.id;
+    listObjectsInS3ForUser(userId).then((objects) => {
+      res.json(objects);
+    });
   });
 }

server/controllers/user.controller.js

@@ -17,7 +17,8 @@ export function userResponse(user) {
     preferences: user.preferences,
     apiKeys: user.apiKeys,
     verified: user.verified,
-    id: user._id
+    id: user._id,
+    totalSize: req.user.totalSize
   };
 }
 

server/migrations/populateTotalSize.js

@@ -0,0 +1,28 @@
+import mongoose from 'mongoose';
+
+import User from '../models/user';
+import { listObjectsInS3ForUser } from '../controllers/aws.controller';
+
+// Connect to MongoDB
+mongoose.Promise = global.Promise;
+mongoose.connect(process.env.MONGO_URL, { useMongoClient: true });
+mongoose.connection.on('error', () => {
+  console.error('MongoDB Connection Error. Please make sure that MongoDB is running.');
+  process.exit(1);
+});
+
+User.find({}, {}, { timeout: true }).cursor().eachAsync((user) => {
+  console.log(user.id);
+  if (user.totalSize !== undefined) {
+    console.log('Already updated size for user: ' + user.username);
+    return Promise.resolve();
+  }
+  return listObjectsInS3ForUser(user.id).then((objects) => {
+    return User.findByIdAndUpdate(user.id, { $set: { totalSize: objects.totalSize } });
+  }).then(() => {
+    console.log('Updated new total size for user: ' + user.username);
+  });
+}).then(() => {
+  console.log('Done iterating over every user');
+  process.exit(0);
+});

server/migrations/start.js

@@ -1,3 +1,7 @@
 require('@babel/register');
 require('@babel/polyfill');
-require('./truncate');
+const path = require('path');
+require('dotenv').config({ path: path.resolve('.env') });
+require('./populateTotalSize');
+// require('./moveBucket');
+// require('./truncate');

server/models/user.js

@@ -66,7 +66,8 @@ const userSchema = new Schema({
     soundOutput: { type: Boolean, default: false },
     theme: { type: String, default: 'light' },
     autorefresh: { type: Boolean, default: false }
-  }
+  },
+  totalSize: { type: Number, default: 0 }
 }, { timestamps: true, usePushEach: true });
 
 /**

server/routes/aws.routes.js

@@ -7,6 +7,6 @@ const router = new Router();
 router.post('/S3/sign', isAuthenticated, AWSController.signS3);
 router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3);
 router.delete('/S3/:object_key', isAuthenticated, AWSController.deleteObjectFromS3);
-router.get('/S3/objects', AWSController.listObjectsInS3ForUser);
+router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);
 
 export default router;