emails: add test command & update backend, config

Bug: T412427

Author: jsnshrmn

TWLight/emails/backends/mediawiki.py

@@ -41,7 +41,7 @@ def conn(*args, **kwargs):
                         logger.warning("ConnectionError exhausted retries")
                         raise e
                     logger.warning(
-                        "ConnectionError, retrying in {self.retry_after_conn}s"
+                        "ConnectionError, retrying in {}s".format(self.retry_after_conn)
                     )
                     sleep(retry_after_conn)
                     continue
@@ -56,20 +56,20 @@ def _handle_maxlag(response):
     data = response.json()
     try:
         if data["error"]["code"] != "maxlag":
-            return data
+            return response
     except KeyError:
-        return data
+        return response
 
     retry_after = float(response.headers.get("Retry-After", 5))
     retry_on_lag_error = 50
     no_retry = 0 <= retry_on_lag_error < try_count
 
     message = "Server exceeded maxlag"
     if not no_retry:
-        message += f", retrying in {retry_after}s"
+        message += ", retrying in {}s".format(retry_after)
     if "lag" in data["error"]:
-        message += f", lag={data['error']['lag']}"
-    message += f", API={self.url}"
+        message += ", lag={}".format(data["error"]["lag"])
+    message += ", API=".format(self.url)
 
     log = logger.warning if no_retry else logger.info
     log(
@@ -117,6 +117,7 @@ def __init__(
         self.email_token = None
         self.session = None
         self._lock = RLock()
+        logger.info("Email connection constructed.")
 
     def open(self):
         """
@@ -164,15 +165,26 @@ def open(self):
                 raise Exception("There was an error in the request for the login.")
 
             # GET request to fetch Email token
+            # see: https://www.mediawiki.org/wiki/API:Emailuser#Token
             email_token_params = {"action": "query", "meta": "tokens", "format": "json"}
 
             logger.info("Getting email token...")
             email_token_response = session.get(url=self.url, params=email_token_params)
+            if email_token_response.status_code != 200:
+                raise Exception(
+                    "There was an error in the request for the email token."
+                )
+
             email_token_data = email_token_response.json()
 
+            email_token = email_token_data["query"]["tokens"]["csrftoken"]
+            if not email_token:
+                raise Exception("There was an error obtaining the email token.")
+
             # Assign the session and email token
-            self.email_token = email_token_data["query"]["tokens"]["csrftoken"]
+            self.email_token = email_token
             self.session = session
+            logger.info("Email API session ready.")
             return True
         except:
             if not self.fail_silently:
@@ -232,10 +244,10 @@ def _send(self, email_message):
                 emailable_response = self.session.post(
                     url=self.url, data=emailable_params
                 )
-                logger.info(emailable_response.text)
                 emailable_data = emailable_response.json()
                 emailable = "emailable" in emailable_data["query"]["users"][0]
                 if not emailable:
+                    logger.warning("User not emailable, email skipped.")
                     continue
 
                 # POST request to send an email
@@ -250,12 +262,17 @@ def _send(self, email_message):
                 }
 
                 logger.info("Sending email...")
-                emailuser_data = _handle_maxlag(
+                emailuser_response = _handle_maxlag(
                     self.session.post(url=self.url, data=email_params)
                 )
-                logger.info(emailuser_data)
+                if emailuser_response.status_code != 200:
+                    raise Exception(
+                        "There was an error in the request to send the email."
+                    )
+                emailuser_data = emailuser_response.json()
                 if emailuser_data["emailuser"]["result"] != "Success":
                     raise Exception("There was an error when trying to send the email.")
+                logger.info("Email sent.")
         except:
             if not self.fail_silently:
                 raise

TWLight/emails/tasks.py

@@ -23,6 +23,7 @@
 """
 
 from djmail import template_mail
+from djmail.models import Message
 from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail
 import logging
 import os
@@ -31,6 +32,7 @@
 from django_comments.models import Comment
 from django_comments.signals import comment_was_posted
 from django.contrib.sites.shortcuts import get_current_site
+from django.core.mail import get_connection
 from django.urls import reverse_lazy
 from django.db.models.signals import pre_save
 from django.dispatch import receiver
@@ -40,8 +42,7 @@
 from TWLight.applications.signals import Reminder
 from TWLight.resources.models import AccessCode, Partner
 from TWLight.users.groups import get_restricted
-from TWLight.users.signals import Notice, Survey, UserLoginRetrieval
-
+from TWLight.users.signals import Notice, Survey, TestEmail, UserLoginRetrieval
 
 logger = logging.getLogger(__name__)
 
@@ -74,6 +75,10 @@ class SurveyActiveUser(template_mail.TemplateMail):
     name = "survey_active_user"
 
 
+class Test(template_mail.TemplateMail):
+    name = "test"
+
+
 class CoordinatorReminderNotification(template_mail.TemplateMail):
     name = "coordinator_reminder_notification"
 
@@ -203,6 +208,17 @@ def send_survey_active_user_emails(sender, **kwargs):
     )
 
 
+@receiver(TestEmail.test)
+def send_test(sender, **kwargs):
+    email = kwargs["email"]
+    connection = get_connection(
+        backend="TWLight.emails.backends.mediawiki.EmailBackend"
+    )
+    template_email = Test()
+    email = template_email.make_email_object(email, {}, connection=connection)
+    email.send()
+
+
 @receiver(comment_was_posted)
 def send_comment_notification_emails(sender, **kwargs):
     """

TWLight/emails/templates/emails/test-body-html.html

@@ -0,0 +1 @@
+<p>Please disregard this message; this is only a test</p>

TWLight/emails/templates/emails/test-body-text.html

@@ -0,0 +1 @@
+Please disregard this message; this is only a test

TWLight/emails/templates/emails/test-subject.html

@@ -0,0 +1 @@
+test message; please disregard

TWLight/settings/base.py

@@ -419,6 +419,13 @@ def show_toolbar(request):
 # ------------------------------------------------------------------------------
 
 TWLIGHT_API_PROVIDER_ENDPOINT = os.environ.get("TWLIGHT_API_PROVIDER_ENDPOINT", None)
+MW_API_URL = os.environ.get("MW_API_URL", None)
+MW_API_REQUEST_TIMEOUT = os.environ.get("MW_API_REQUEST_TIMEOUT", 60)
+MW_API_REQUEST_DELAY = os.environ.get("MW_API_REQUEST_DELAY", 0)
+MW_API_REQUEST_RETRY_DELAY = os.environ.get("MW_API_REQUEST_RETRY_DELAY", 5)
+MW_API_MAXLAG = os.environ.get("MW_API_MAXLAG", 5)
+MW_API_EMAIL_USER = os.environ.get("MW_API_EMAIL_USER", None)
+MW_API_EMAIL_PASSWORD = os.environ.get("MW_API_EMAIL_PASSWORD", None)
 
 # COMMENTS CONFIGURATION
 # ------------------------------------------------------------------------------

TWLight/users/management/commands/test_email.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+from TWLight.users.signals import TestEmail
+
+
+class Command(BaseCommand):
+    help = "Sends testmail to a wikipedia editor."
+
+    def add_arguments(self, parser):
+        parser.add_argument('wp_username', type=str, help='The wikipedia editor to send the test email to')
+
+    def handle(self, *args, **options):
+        user = User.objects.select_related("editor").get(editor__wp_username=options["wp_username"])
+        TestEmail.test.send(
+            sender=self.__class__,
+            wp_username=user.editor.wp_username,
+            email=user.email,
+        )

TWLight/users/signals.py

@@ -29,6 +29,10 @@ class Survey(object):
     survey_active_user = Signal()
 
 
+class TestEmail(object):
+    test = Signal()
+
+
 class UserLoginRetrieval(object):
     user_retrieve_monthly_logins = Signal()
 

conf/local.twlight.env

@@ -1,5 +1,6 @@
 DJANGO_DB_HOST=db
-DJANGO_EMAIL_BACKEND=django_dkim.backends.console.EmailBackend
+#DJANGO_EMAIL_BACKEND=TWLight.emails.backends.mediawiki.EmailBackend
+DJANGO_EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend
 DJANGO_EMAIL_ADMINS_BACKEND=django.core.mail.backends.dummy.EmailBackend
 DKIM_SELECTOR=local
 DKIM_DOMAIN=localhost
@@ -19,5 +20,6 @@ DEBUG=True
 TWLIGHT_OAUTH_PROVIDER_URL=https://meta.wikimedia.org/w/index.php
 TWLIGHT_API_PROVIDER_ENDPOINT=https://meta.wikimedia.org/w/api.php
 TWLIGHT_EZPROXY_URL=https://ezproxy.dev.localdomain:2443
+MW_API_URL=http://host.docker.internal:8080/w/api.php
 # seeem to be having troubles with --workers > 1
 GUNICORN_CMD_ARGS=--name twlight --workers 1 --backlog 2048 --timeout 300 --bind=0.0.0.0:80 --forwarded-allow-ips * --reload --log-level=info

docker-compose.override.yml

@@ -25,9 +25,14 @@ secrets:
 
 services:
   twlight:
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     image: quay.io/wikipedialibrary/twlight:local
     env_file:
-      - ./conf/local.twlight.env
+      - path: ./conf/local.twlight.env
+        required: true
+      - path: .env
+        required: false
     # Local environment should mount things from the code directory
     volumes:
       - type: bind