Remove old mailer and add Redis based queue system (#736)

* Add BullMQ and send mails to redis queue

* Skip tests for now

* Include new mailer und configuration

* Fix mail tests

* Log addition to queue

* Fix lint errors

Author: mpfeil

docker-compose.yml

@@ -16,4 +16,3 @@ services:
       - ./.scripts/mongodb/osem_admin.sh:/docker-entrypoint-initdb.d/osem_admin.sh
       # - ./.scripts/mongodb/osem_seed_boxes.sh:/docker-entrypoint-initdb.d/osem_seed_boxes.sh
       # - ./.scripts/mongodb/osem_seed_measurements.sh:/docker-entrypoint-initdb.d/osem_seed_measurements.sh
-      

packages/api/lib/controllers/usersController.js

@@ -2,10 +2,19 @@
 
 const { User } = require('@sensebox/opensensemap-api-models'),
   { InternalServerError, ForbiddenError } = require('restify-errors'),
-  { checkContentType, redactEmail, clearCache, postToMattermost } = require('../helpers/apiUtils'),
+  {
+    checkContentType,
+    redactEmail,
+    clearCache,
+    postToMattermost,
+  } = require('../helpers/apiUtils'),
   { retrieveParameters } = require('../helpers/userParamHelpers'),
   handleError = require('../helpers/errorHandler'),
-  { createToken, refreshJwt, invalidateToken } = require('../helpers/jwtHelpers');
+  {
+    createToken,
+    refreshJwt,
+    invalidateToken,
+  } = require('../helpers/jwtHelpers');
 
 /**
  * define for nested user parameter for box creation request
@@ -47,16 +56,27 @@ const registerUser = async function registerUser (req, res) {
   const { email, password, language, name } = req._userParams;
 
   try {
-    const newUser = await new User({ name, email, password, language })
-      .save();
-    postToMattermost(`New User: ${newUser.name} (${redactEmail(newUser.email)})`);
+    const newUser = await new User({ name, email, password, language }).save();
+    postToMattermost(
+      `New User: ${newUser.name} (${redactEmail(newUser.email)})`
+    );
 
     try {
       const { token, refreshToken } = await createToken(newUser);
 
-      return res.send(201, { code: 'Created', message: 'Successfully registered new user', data: { user: newUser }, token, refreshToken });
+      return res.send(201, {
+        code: 'Created',
+        message: 'Successfully registered new user',
+        data: { user: newUser },
+        token,
+        refreshToken,
+      });
     } catch (err) {
-      return Promise.reject(new InternalServerError(`User successfully created but unable to create jwt token: ${err.message}`));
+      return Promise.reject(
+        new InternalServerError(
+          `User successfully created but unable to create jwt token: ${err.message}`
+        )
+      );
     }
   } catch (err) {
     return handleError(err);
@@ -82,18 +102,26 @@ const signIn = async function signIn (req, res) {
 
   try {
     // lowercase for email
-    const user = await User
-      .findOne({ $or: [{ email: emailOrName.toLowerCase() }, { name: emailOrName }] })
-      .exec();
+    const user = await User.findOne({
+      $or: [{ email: emailOrName.toLowerCase() }, { name: emailOrName }],
+    }).exec();
 
     if (!user) {
-      return Promise.reject(new ForbiddenError('User and or password not valid!'));
+      return Promise.reject(
+        new ForbiddenError('User and or password not valid!')
+      );
     }
 
     if (await user.checkPassword(password)) {
       const { token, refreshToken } = await createToken(user);
 
-      return res.send(200, { code: 'Authorized', message: 'Successfully signed in', data: { user }, token, refreshToken });
+      return res.send(200, {
+        code: 'Authorized',
+        message: 'Successfully signed in',
+        data: { user },
+        token,
+        refreshToken,
+      });
     }
   } catch (err) {
     if (err.name === 'ModelError' && err.message === 'Password incorrect') {
@@ -119,8 +147,16 @@ const signIn = async function signIn (req, res) {
  */
 const refreshJWT = async function refreshJWT (req, res) {
   try {
-    const { token, refreshToken, user } = await refreshJwt(req._userParams.token);
-    res.send(200, { code: 'Authorized', message: 'Successfully refreshed auth', data: { user }, token, refreshToken });
+    const { token, refreshToken, user } = await refreshJwt(
+      req._userParams.token
+    );
+    res.send(200, {
+      code: 'Authorized',
+      message: 'Successfully refreshed auth',
+      data: { user },
+      token,
+      refreshToken,
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -174,7 +210,11 @@ const requestResetPassword = async function requestResetPassword (req, res) {
 const resetPassword = async function resetPassword (req, res) {
   try {
     await User.resetPassword(req._userParams);
-    res.send(200, { code: 'Ok', message: 'Password successfully changed. You can now login with your new password' });
+    res.send(200, {
+      code: 'Ok',
+      message:
+        'Password successfully changed. You can now login with your new password',
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -193,7 +233,10 @@ const resetPassword = async function resetPassword (req, res) {
 const confirmEmailAddress = async function confirmEmailAddress (req, res) {
   try {
     await User.confirmEmail(req._userParams);
-    res.send(200, { code: 'Ok', message: 'E-Mail successfully confirmed. Thank you' });
+    res.send(200, {
+      code: 'Ok',
+      message: 'E-Mail successfully confirmed. Thank you',
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -211,7 +254,10 @@ const getUserBoxes = async function getUserBoxes (req, res) {
   try {
     const boxes = await req.user.getBoxes();
     const sharedBoxes = await req.user.getSharedBoxes();
-    res.send(200, { code: 'Ok', data: { boxes: boxes, sharedBoxes: sharedBoxes } });
+    res.send(200, {
+      code: 'Ok',
+      data: { boxes: boxes, sharedBoxes: sharedBoxes },
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -242,15 +288,23 @@ const getUser = async function getUser (req, res) {
  */
 const updateUser = async function updateUser (req, res) {
   try {
-    const { updated, signOut, messages, updatedUser } = await req.user.updateUser(req._userParams);
+    const { updated, signOut, messages, updatedUser } =
+      await req.user.updateUser(req._userParams);
     if (updated === false) {
-      return res.send(200, { code: 'Ok', message: 'No changed properties supplied. User remains unchanged.' });
+      return res.send(200, {
+        code: 'Ok',
+        message: 'No changed properties supplied. User remains unchanged.',
+      });
     }
 
     if (signOut === true) {
       invalidateToken(req);
     }
-    res.send(200, { code: 'Ok', message: `User successfully saved.${messages.join('.')}`, data: { me: updatedUser } });
+    res.send(200, {
+      code: 'Ok',
+      message: `User successfully saved.${messages.join('.')}`,
+      data: { me: updatedUser },
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -273,9 +327,14 @@ const deleteUser = async function deleteUser (req, res) {
     invalidateToken(req);
 
     await req.user.destroyUser();
-    res.send(200, { code: 'Ok', message: 'User and all boxes of user marked for deletion. Bye Bye!' });
+    res.send(200, {
+      code: 'Ok',
+      message: 'User and all boxes of user marked for deletion. Bye Bye!',
+    });
     clearCache(['getBoxes', 'getStats']);
-    postToMattermost(`User deleted: ${req.user.name} (${redactEmail(req.user.email)})`);
+    postToMattermost(
+      `User deleted: ${req.user.name} (${redactEmail(req.user.email)})`
+    );
   } catch (err) {
     return handleError(err);
   }
@@ -290,14 +349,20 @@ const deleteUser = async function deleteUser (req, res) {
  * @apiSuccess {String} code `Ok`
  * @apiSuccess {String} message `Email confirmation has been sent to <emailaddress>`
  */
-const requestEmailConfirmation = async function requestEmailConfirmation (req, res) {
+const requestEmailConfirmation = async function requestEmailConfirmation (
+  req,
+  res
+) {
   try {
     const result = await req.user.resendEmailConfirmation();
     let usedAddress = result.email;
     if (result.unconfirmedEmail) {
       usedAddress = result.unconfirmedEmail;
     }
-    res.send(200, { code: 'Ok', message: `Email confirmation has been sent to ${usedAddress}` });
+    res.send(200, {
+      code: 'Ok',
+      message: `Email confirmation has been sent to ${usedAddress}`,
+    });
   } catch (err) {
     return handleError(err);
   }
@@ -310,41 +375,39 @@ module.exports = {
       { name: 'email', dataType: 'email', required: true },
       { predef: 'password' },
       { name: 'name', required: true, dataType: 'as-is' },
-      { name: 'language', defaultValue: 'en_US' }
+      { name: 'language', defaultValue: 'en_US' },
     ]),
-    registerUser
+    registerUser,
   ],
   signIn: [
     checkContentType,
     retrieveParameters([
       { name: 'email', required: true },
       { predef: 'password' },
     ]),
-    signIn
+    signIn,
   ],
   signOut,
   resetPassword: [
     checkContentType,
     retrieveParameters([
       { name: 'token', required: true },
-      { predef: 'password' }
+      { predef: 'password' },
     ]),
-    resetPassword
+    resetPassword,
   ],
   requestResetPassword: [
     checkContentType,
-    retrieveParameters([
-      { name: 'email', dataType: 'email', required: true }
-    ]),
-    requestResetPassword
+    retrieveParameters([{ name: 'email', dataType: 'email', required: true }]),
+    requestResetPassword,
   ],
   confirmEmailAddress: [
     checkContentType,
     retrieveParameters([
       { name: 'token', required: true },
       { name: 'email', dataType: 'email', required: true },
     ]),
-    confirmEmailAddress
+    confirmEmailAddress,
   ],
   requestEmailConfirmation,
   getUserBoxes,
@@ -355,23 +418,19 @@ module.exports = {
       { predef: 'password', name: 'currentPassword', required: false },
       { predef: 'password', name: 'newPassword', required: false },
       { name: 'name', dataType: 'as-is' },
-      { name: 'language' }
+      { name: 'language' },
     ]),
-    updateUser
+    updateUser,
   ],
   getUser,
   refreshJWT: [
     checkContentType,
-    retrieveParameters([
-      { name: 'token', required: true }
-    ]),
-    refreshJWT
+    retrieveParameters([{ name: 'token', required: true }]),
+    refreshJWT,
   ],
   deleteUser: [
     checkContentType,
-    retrieveParameters([
-      { predef: 'password' }
-    ]),
-    deleteUser
-  ]
+    retrieveParameters([{ predef: 'password' }]),
+    deleteUser,
+  ],
 };

packages/models/index.js

@@ -7,36 +7,44 @@ process.env.SUPPRESS_NO_CONFIG_WARNING = 'y';
 const config = require('config');
 
 config.util.setModuleDefaults('openSenseMap-API-models', {
-  'db': {
-    'host': 'localhost',
-    'port': 27017,
-    'user': 'admin',
-    'userpass': 'admin',
-    'authsource': 'OSeM-api',
-    'db': 'OSeM-api',
-    'mongo_uri': ''
+  db: {
+    host: 'localhost',
+    port: 27017,
+    user: 'admin',
+    userpass: 'admin',
+    authsource: 'OSeM-api',
+    db: 'OSeM-api',
+    mongo_uri: '',
   },
-  'integrations': {
-    'ca_cert': '',
-    'cert': '',
-    'key': '',
-    'mailer': {
-      'url': '',
-      'origin': ''
+  integrations: {
+    ca_cert: '',
+    cert: '',
+    key: '',
+    redis: {
+      host: '',
+      port: 6379,
+      username: '',
+      password: '',
+      db: 0,
+    },
+    mailer: {
+      url: '',
+      origin: '',
+      queue: 'mails',
+    },
+    mqtt: {
+      url: '',
     },
-    'mqtt': {
-      'url': ''
-    }
   },
-  'password': {
-    'min_length': 8,
-    'salt_factor': 13
+  password: {
+    min_length: 8,
+    salt_factor: 13,
   },
-  'claims_ttl': {
-    'amount': 1,
-    'unit': 'd'
+  claims_ttl: {
+    amount: 1,
+    unit: 'd',
   },
-  'image_folder': './userimages/',
+  image_folder: './userimages/',
 });
 
 const { model: Box } = require('./src/box/box'),

packages/models/package.json

@@ -10,6 +10,7 @@
     "@sensebox/osem-protos": "^1.1.0",
     "@sensebox/sketch-templater": "1.13.0",
     "bcrypt": "^5.1.0",
+    "bullmq": "^3.10.1",
     "config": "^3.3.6",
     "got": "^11.8.2",
     "isemail": "^3.0.0",