Add SparkPost support (#20)

Implement SparkPost backend and tracking webhooks.

Closes #11.

Author: medmunds

.travis.yml

@@ -26,6 +26,8 @@ cache:
 install:
   - pip install --upgrade setuptools pip
   - pip install $DJANGO
-  - pip install .
+  # For now, install all ESPs and test at once
+  # (in future, might want to matrix ESPs to test cross-dependencies)
+  - pip install .[mailgun,mandrill,postmark,sendgrid,sparkpost]
   - pip list
 script: python setup.py test

README.rst

@@ -1,5 +1,5 @@
-Anymail: Django email backends for Mailgun, Postmark, SendGrid and more
-=======================================================================
+Anymail: Django email backends for Mailgun, Postmark, SendGrid, SparkPost and more
+==================================================================================
 
  **EARLY DEVELOPMENT**
 
@@ -30,7 +30,7 @@ Anymail integrates several transactional email service providers (ESPs) into Dja
 with a consistent API that lets you use ESP-added features without locking your code
 to a particular ESP.
 
-It currently fully supports Mailgun, Postmark, and SendGrid,
+It currently fully supports Mailgun, Postmark, SendGrid, and SparkPost,
 and has limited support for Mandrill.
 
 Anymail normalizes ESP functionality so it "just works" with Django's
@@ -78,13 +78,16 @@ Anymail 1-2-3
 .. This quickstart section is also included in docs/quickstart.rst
 
 This example uses Mailgun, but you can substitute Postmark or SendGrid
-or any other supported ESP where you see "mailgun":
+or SparkPost or any other supported ESP where you see "mailgun":
 
 1. Install Anymail from PyPI:
 
    .. code-block:: console
 
-        $ pip install django-anymail
+        $ pip install django-anymail[mailgun]
+
+   (The `[mailgun]` part installs any additional packages needed for that ESP.
+   Mailgun doesn't have any, but some other ESPs do.)
 
 
 2. Edit your project's ``settings.py``:

anymail/backends/sparkpost.py

@@ -0,0 +1,190 @@
+from __future__ import absolute_import  # we want the sparkpost package, not our own module
+
+from .base import AnymailBaseBackend, BasePayload
+from ..exceptions import AnymailAPIError, AnymailImproperlyInstalled, AnymailConfigurationError
+from ..message import AnymailRecipientStatus
+from ..utils import get_anymail_setting
+
+try:
+    from sparkpost import SparkPost, SparkPostException
+except ImportError:
+    raise AnymailImproperlyInstalled(missing_package='sparkpost', backend='sparkpost')
+
+
+class SparkPostBackend(AnymailBaseBackend):
+    """
+    SparkPost Email Backend (using python-sparkpost client)
+    """
+
+    def __init__(self, **kwargs):
+        """Init options from Django settings"""
+        super(SparkPostBackend, self).__init__(**kwargs)
+        # SPARKPOST_API_KEY is optional - library reads from env by default
+        self.api_key = get_anymail_setting('api_key', esp_name=self.esp_name,
+                                           kwargs=kwargs, allow_bare=True, default=None)
+        try:
+            self.sp = SparkPost(self.api_key)  # SparkPost API instance
+        except SparkPostException as err:
+            # This is almost certainly a missing API key
+            raise AnymailConfigurationError(
+                "Error initializing SparkPost: %s\n"
+                "You may need to set ANYMAIL = {'SPARKPOST_API_KEY': ...} "
+                "or ANYMAIL_SPARKPOST_API_KEY in your Django settings, "
+                "or SPARKPOST_API_KEY in your environment." % str(err)
+            )
+
+    # Note: SparkPost python API doesn't expose requests session sharing
+    # (so there's no need to implement open/close connection management here)
+
+    def build_message_payload(self, message, defaults):
+        return SparkPostPayload(message, defaults, self)
+
+    def post_to_esp(self, payload, message):
+        params = payload.get_api_params()
+        try:
+            response = self.sp.transmissions.send(**params)
+        except SparkPostException as err:
+            raise AnymailAPIError(
+                str(err), backend=self, email_message=message, payload=payload,
+                response=getattr(err, 'response', None),  # SparkPostAPIException requests.Response
+                status_code=getattr(err, 'status', None),  # SparkPostAPIException HTTP status_code
+            )
+        return response
+
+    def parse_recipient_status(self, response, payload, message):
+        try:
+            accepted = response['total_accepted_recipients']
+            rejected = response['total_rejected_recipients']
+            transmission_id = response['id']
+        except (KeyError, TypeError) as err:
+            raise AnymailAPIError(
+                "%s in SparkPost.transmissions.send result %r" % (str(err), response),
+                backend=self, email_message=message, payload=payload,
+            )
+
+        # SparkPost doesn't (yet*) tell us *which* recipients were accepted or rejected.
+        # (* looks like undocumented 'rcpt_to_errors' might provide this info.)
+        # If all are one or the other, we can report a specific status;
+        # else just report 'unknown' for all recipients.
+        recipient_count = len(payload.all_recipients)
+        if accepted == recipient_count and rejected == 0:
+            status = 'queued'
+        elif rejected == recipient_count and accepted == 0:
+            status = 'rejected'
+        else:  # mixed results, or wrong total
+            status = 'unknown'
+        recipient_status = AnymailRecipientStatus(message_id=transmission_id, status=status)
+        return {recipient.email: recipient_status for recipient in payload.all_recipients}
+
+
+class SparkPostPayload(BasePayload):
+    def init_payload(self):
+        self.params = {}
+        self.all_recipients = []
+        self.to_emails = []
+        self.merge_data = {}
+
+    def get_api_params(self):
+        # Compose recipients param from to_emails and merge_data (if any)
+        recipients = []
+        for email in self.to_emails:
+            rcpt = {'address': {'email': email.email}}
+            if email.name:
+                rcpt['address']['name'] = email.name
+            try:
+                rcpt['substitution_data'] = self.merge_data[email.email]
+            except KeyError:
+                pass  # no merge_data or none for this recipient
+            recipients.append(rcpt)
+        if recipients:
+            self.params['recipients'] = recipients
+
+        return self.params
+
+    def set_from_email(self, email):
+        self.params['from_email'] = email.address
+
+    def set_to(self, emails):
+        if emails:
+            self.to_emails = emails  # bound to params['recipients'] in get_api_params
+            self.all_recipients += emails
+
+    def set_cc(self, emails):
+        if emails:
+            self.params['cc'] = [email.address for email in emails]
+            self.all_recipients += emails
+
+    def set_bcc(self, emails):
+        if emails:
+            self.params['bcc'] = [email.address for email in emails]
+            self.all_recipients += emails
+
+    def set_subject(self, subject):
+        self.params['subject'] = subject
+
+    def set_reply_to(self, emails):
+        if emails:
+            # reply_to is only documented as a single email, but this seems to work:
+            self.params['reply_to'] = ', '.join([email.address for email in emails])
+
+    def set_extra_headers(self, headers):
+        if headers:
+            self.params['custom_headers'] = headers
+
+    def set_text_body(self, body):
+        self.params['text'] = body
+
+    def set_html_body(self, body):
+        if 'html' in self.params:
+            # second html body could show up through multiple alternatives, or html body + alternative
+            self.unsupported_feature("multiple html parts")
+        self.params['html'] = body
+
+    def add_attachment(self, attachment):
+        if attachment.inline:
+            param = 'inline_images'
+            name = attachment.cid
+        else:
+            param = 'attachments'
+            name = attachment.name or ''
+
+        self.params.setdefault(param, []).append({
+            'type': attachment.mimetype,
+            'name': name,
+            'data': attachment.b64content})
+
+    # Anymail-specific payload construction
+    def set_metadata(self, metadata):
+        self.params['metadata'] = metadata
+
+    def set_send_at(self, send_at):
+        try:
+            self.params['start_time'] = send_at.replace(microsecond=0).isoformat()
+        except (AttributeError, TypeError):
+            self.params['start_time'] = send_at  # assume user already formatted
+
+    def set_tags(self, tags):
+        if len(tags) > 0:
+            self.params['campaign'] = tags[0]
+            if len(tags) > 1:
+                self.unsupported_feature('multiple tags (%r)' % tags)
+
+    def set_track_clicks(self, track_clicks):
+        self.params['track_clicks'] = track_clicks
+
+    def set_track_opens(self, track_opens):
+        self.params['track_opens'] = track_opens
+
+    def set_template_id(self, template_id):
+        # 'template' transmissions.send param becomes 'template_id' in API json 'content'
+        self.params['template'] = template_id
+
+    def set_merge_data(self, merge_data):
+        self.merge_data = merge_data  # merged into params['recipients'] in get_api_params
+
+    def set_merge_global_data(self, merge_global_data):
+        self.params['substitution_data'] = merge_global_data
+
+    # ESP-specific payload construction
+    def set_esp_extra(self, extra):
+        self.params.update(extra)

anymail/urls.py

@@ -4,6 +4,7 @@
 from .webhooks.mandrill import MandrillTrackingWebhookView
 from .webhooks.postmark import PostmarkTrackingWebhookView
 from .webhooks.sendgrid import SendGridTrackingWebhookView
+from .webhooks.sparkpost import SparkPostTrackingWebhookView
 
 
 app_name = 'anymail'
@@ -12,4 +13,5 @@
     url(r'^mandrill/tracking/$', MandrillTrackingWebhookView.as_view(), name='mandrill_tracking_webhook'),
     url(r'^postmark/tracking/$', PostmarkTrackingWebhookView.as_view(), name='postmark_tracking_webhook'),
     url(r'^sendgrid/tracking/$', SendGridTrackingWebhookView.as_view(), name='sendgrid_tracking_webhook'),
+    url(r'^sparkpost/tracking/$', SparkPostTrackingWebhookView.as_view(), name='sparkpost_tracking_webhook'),
 ]

anymail/webhooks/sparkpost.py

@@ -0,0 +1,136 @@
+import json
+from datetime import datetime
+
+from django.utils.timezone import utc
+
+from .base import AnymailBaseWebhookView
+from ..exceptions import AnymailConfigurationError
+from ..signals import tracking, AnymailTrackingEvent, EventType, RejectReason
+
+
+class SparkPostBaseWebhookView(AnymailBaseWebhookView):
+    """Base view class for SparkPost webhooks"""
+
+    def parse_events(self, request):
+        raw_events = json.loads(request.body.decode('utf-8'))
+        unwrapped_events = [self.unwrap_event(raw_event) for raw_event in raw_events]
+        return [
+            self.esp_to_anymail_event(event_class, event, raw_event)
+            for (event_class, event, raw_event) in unwrapped_events
+            if event is not None  # filter out empty "ping" events
+        ]
+
+    def unwrap_event(self, raw_event):
+        """Unwraps SparkPost event structure, and returns event_class, event, raw_event
+
+        raw_event is of form {'msys': {event_class: {...event...}}}
+
+        Can return None, None, raw_event for SparkPost "ping" raw_event={'msys': {}}
+        """
+        event_classes = raw_event['msys'].keys()
+        try:
+            (event_class,) = event_classes
+            event = raw_event['msys'][event_class]
+        except ValueError:  # too many/not enough event_classes to unpack
+            if len(event_classes) == 0:
+                # Empty event (SparkPost sometimes sends as a "ping")
+                event_class = event = None
+            else:
+                raise TypeError("Invalid SparkPost webhook event has multiple event classes: %r" % raw_event)
+        return event_class, event, raw_event
+
+    def esp_to_anymail_event(self, event_class, event, raw_event):
+        raise NotImplementedError()
+
+
+class SparkPostTrackingWebhookView(SparkPostBaseWebhookView):
+    """Handler for SparkPost message, engagement, and generation event webhooks"""
+
+    signal = tracking
+
+    event_types = {
+        # Map SparkPost event.type: Anymail normalized type
+        'bounce': EventType.BOUNCED,
+        'delivery': EventType.DELIVERED,
+        'injection': EventType.QUEUED,
+        'spam_complaint': EventType.COMPLAINED,
+        'out_of_band': EventType.BOUNCED,
+        'policy_rejection': EventType.REJECTED,
+        'delay': EventType.DEFERRED,
+        'click': EventType.CLICKED,
+        'open': EventType.OPENED,
+        'generation_failure': EventType.FAILED,
+        'generation_rejection': EventType.REJECTED,
+        'list_unsubscribe': EventType.UNSUBSCRIBED,
+        'link_unsubscribe': EventType.UNSUBSCRIBED,
+    }
+
+    reject_reasons = {
+        # Map SparkPost event.bounce_class: Anymail normalized reject reason.
+        # Can also supply (RejectReason, EventType) for bounce_class that affects our event_type.
+        # https://support.sparkpost.com/customer/portal/articles/1929896
+        '1': RejectReason.OTHER,     # Undetermined (response text could not be identified)
+        '10': RejectReason.INVALID,  # Invalid Recipient
+        '20': RejectReason.BOUNCED,  # Soft Bounce
+        '21': RejectReason.BOUNCED,  # DNS Failure
+        '22': RejectReason.BOUNCED,  # Mailbox Full
+        '23': RejectReason.BOUNCED,  # Too Large
+        '24': RejectReason.TIMED_OUT,  # Timeout
+        '25': RejectReason.BLOCKED,  # Admin Failure (configured policies)
+        '30': RejectReason.BOUNCED,  # Generic Bounce: No RCPT
+        '40': RejectReason.BOUNCED,  # Generic Bounce: unspecified reasons
+        '50': RejectReason.BLOCKED,  # Mail Block (by the receiver)
+        '51': RejectReason.SPAM,     # Spam Block (by the receiver)
+        '52': RejectReason.SPAM,     # Spam Content (by the receiver)
+        '53': RejectReason.OTHER,    # Prohibited Attachment (by the receiver)
+        '54': RejectReason.BLOCKED,  # Relaying Denied (by the receiver)
+        '60': (RejectReason.OTHER, EventType.AUTORESPONDED),  # Auto-Reply/vacation
+        '70': RejectReason.BOUNCED,  # Transient Failure
+        '80': (RejectReason.OTHER, EventType.SUBSCRIBED),  # Subscribe
+        '90': (RejectReason.UNSUBSCRIBED, EventType.UNSUBSCRIBED),  # Unsubscribe
+        '100': (RejectReason.OTHER, EventType.AUTORESPONDED),  # Challenge-Response
+    }
+
+    def esp_to_anymail_event(self, event_class, event, raw_event):
+        if event_class == 'relay_event':
+            # This is an inbound event
+            raise AnymailConfigurationError(
+                "You seem to have set SparkPost's *inbound* relay webhook URL "
+                "to Anymail's SparkPost *tracking* webhook URL.")
+
+        event_type = self.event_types.get(event['type'], EventType.UNKNOWN)
+        try:
+            timestamp = datetime.fromtimestamp(int(event['timestamp']), tz=utc)
+        except (KeyError, TypeError, ValueError):
+            timestamp = None
+
+        try:
+            tag = event['campaign_id']  # not 'rcpt_tags' -- those don't come from sending a message
+            tags = [tag] if tag else None
+        except KeyError:
+            tags = None
+
+        try:
+            reject_reason = self.reject_reasons.get(event['bounce_class'], RejectReason.OTHER)
+            try:  # unpack (RejectReason, EventType) for reasons that change our event type
+                reject_reason, event_type = reject_reason
+            except ValueError:
+                pass
+        except KeyError:
+            reject_reason = None  # no bounce_class
+
+        return AnymailTrackingEvent(
+            event_type=event_type,
+            timestamp=timestamp,
+            message_id=event.get('transmission_id', None),  # not 'message_id' -- see SparkPost backend
+            event_id=event.get('event_id', None),
+            recipient=event.get('raw_rcpt_to', None),  # preserves email case (vs. 'rcpt_to')
+            reject_reason=reject_reason,
+            mta_response=event.get('raw_reason', None),
+            # description=???,
+            tags=tags,
+            metadata=event.get('rcpt_meta', None) or None,  # message + recipient metadata
+            click_url=event.get('target_link_url', None),
+            user_agent=event.get('user_agent', None),
+            esp_event=raw_event,
+        )