Postmark inbound: improve inbound parsing

- Support Postmark's RawEmail option;
  recommend it in docs
- Handle Bcc when provided by Postmark
- Obtain `envelope_sender` from Return-Path info
  Postmark now adds, rather than parsing Received-SPF

Related:
- Add `AnymailInboundMessage.bcc` convenience prop
- Test against full Postmark "check" inbound payloads
  (which don't match their docs or real inbound payloads)
- Don't warn about receiving "check" payload

Author: medmunds

CHANGELOG.rst

@@ -59,6 +59,19 @@ Breaking changes
 
 * Require urllib3 1.25 or later (released 2019-04-29).
 
+Features
+~~~~~~~~
+
+* **Postmark inbound:**
+
+  * Handle Postmark's "Include raw email content in JSON payload"
+    inbound option. Enabling this setting is recommended to get
+    the most accurate representation of any received email.
+  * Obtain ``envelope_sender`` from *Return-Path* Postmark now provides.
+    (Replaces potentially faulty *Received-SPF* header parsing.)
+  * Add *Bcc* header to inbound message if provided. Postmark adds bcc
+    when the delivered-to address does not appear in the *To* header.
+
 Other
 ~~~~~
 

anymail/inbound.py

@@ -71,6 +71,12 @@ def cc(self):
         # equivalent to Python 3.2+ message['Cc'].addresses
         return self.get_address_header("Cc")
 
+    @property
+    def bcc(self):
+        """list of EmailAddress objects from Bcc header"""
+        # equivalent to Python 3.2+ message['Bcc'].addresses
+        return self.get_address_header("Bcc")
+
     @property
     def subject(self):
         """str value of Subject header, or None"""
@@ -233,6 +239,7 @@ def construct(
         from_email=None,
         to=None,
         cc=None,
+        bcc=None,
         subject=None,
         headers=None,
         text=None,
@@ -252,6 +259,7 @@ def construct(
         :param from_email: {str|None} value for From header
         :param to: {str|None} value for To header
         :param cc: {str|None} value for Cc header
+        :param bcc: {str|None} value for Bcc header
         :param subject: {str|None} value for Subject header
         :param headers: {sequence[(str, str)]|mapping|None} additional headers
         :param text: {str|None} plaintext body
@@ -279,6 +287,9 @@ def construct(
         if cc is not None:
             del msg["Cc"]
             msg["Cc"] = cc
+        if bcc is not None:
+            del msg["Bcc"]
+            msg["Bcc"] = bcc
         if subject is not None:
             del msg["Subject"]
             msg["Subject"] = subject

anymail/webhooks/postmark.py

@@ -1,9 +1,9 @@
 import json
-import warnings
+from email.utils import unquote
 
 from django.utils.dateparse import parse_datetime
 
-from ..exceptions import AnymailConfigurationError, AnymailWarning
+from ..exceptions import AnymailConfigurationError
 from ..inbound import AnymailInboundMessage
 from ..signals import (
     AnymailInboundEvent,
@@ -161,77 +161,65 @@ class PostmarkInboundWebhookView(PostmarkBaseWebhookView):
     signal = inbound
 
     def esp_to_anymail_event(self, esp_event):
-        if esp_event.get("RecordType", "Inbound") != "Inbound":
+        # Check correct webhook (inbound events don't have RecordType):
+        esp_record_type = esp_event.get("RecordType", "Inbound")
+        if esp_record_type != "Inbound":
             raise AnymailConfigurationError(
-                "You seem to have set Postmark's *%s* webhook "
-                "to Anymail's Postmark *inbound* webhook URL." % esp_event["RecordType"]
+                f"You seem to have set Postmark's *{esp_record_type}* webhook"
+                f" to Anymail's Postmark *inbound* webhook URL."
             )
 
-        attachments = [
-            AnymailInboundMessage.construct_attachment(
-                content_type=attachment["ContentType"],
-                content=(
-                    attachment.get("Content")
-                    # WORKAROUND:
-                    # The test webhooks are not like their real webhooks
-                    # This allows the test webhooks to be parsed.
-                    or attachment["Data"]
-                ),
-                base64=True,
-                filename=attachment.get("Name", "") or None,
-                content_id=attachment.get("ContentID", "") or None,
-            )
-            for attachment in esp_event.get("Attachments", [])
-        ]
-
-        # Warning to the user regarding the workaround of above.
-        for attachment in esp_event.get("Attachments", []):
-            if "Data" in attachment:
-                warnings.warn(
-                    "Received a test webhook attachment. "
-                    "It is recommended to test with real inbound events. "
-                    "See https://github.com/anymail/django-anymail/issues/304 "
-                    "for more information.",
-                    AnymailWarning,
-                )
-                break
-
-        message = AnymailInboundMessage.construct(
-            from_email=self._address(esp_event.get("FromFull")),
-            to=", ".join([self._address(to) for to in esp_event.get("ToFull", [])]),
-            cc=", ".join([self._address(cc) for cc in esp_event.get("CcFull", [])]),
-            # bcc? Postmark specs this for inbound events,
-            # but it's unclear how it could occur
-            subject=esp_event.get("Subject", ""),
-            headers=[
-                (header["Name"], header["Value"])
-                for header in esp_event.get("Headers", [])
-            ],
-            text=esp_event.get("TextBody", ""),
-            html=esp_event.get("HtmlBody", ""),
-            attachments=attachments,
-        )
+        headers = esp_event.get("Headers", [])
+
+        # Postmark inbound prepends "Return-Path" to Headers list
+        # (but it doesn't appear in original message or RawEmail).
+        # (A Return-Path anywhere else in the headers or RawEmail
+        # can't be considered legitimate.)
+        envelope_sender = None
+        if len(headers) > 0 and headers[0]["Name"].lower() == "return-path":
+            envelope_sender = unquote(headers[0]["Value"])  # remove <>
+            headers = headers[1:]  # don't include in message construction
+
+        if "RawEmail" in esp_event:
+            message = AnymailInboundMessage.parse_raw_mime(esp_event["RawEmail"])
+            # Postmark provides Bcc when delivered-to is not in To header,
+            # but doesn't add it to the RawEmail.
+            if esp_event.get("BccFull") and "Bcc" not in message:
+                message["Bcc"] = self._addresses(esp_event["BccFull"])
 
-        # Postmark strips these headers and provides them as separate event fields:
-        if "Date" in esp_event and "Date" not in message:
-            message["Date"] = esp_event["Date"]
-        if "ReplyTo" in esp_event and "Reply-To" not in message:
-            message["Reply-To"] = esp_event["ReplyTo"]
-
-        # Postmark doesn't have a separate envelope-sender field, but it can
-        # be extracted from the Received-SPF header that Postmark will have added.
-        # (More than one Received-SPF? someone's up to something weird?)
-        if len(message.get_all("Received-SPF", [])) == 1:
-            received_spf = message["Received-SPF"].lower()
-            if received_spf.startswith(  # not fail/softfail
-                "pass"
-            ) or received_spf.startswith("neutral"):
-                message.envelope_sender = message.get_param(
-                    "envelope-from", None, header="Received-SPF"
+        else:
+            # RawEmail not included in payload; construct from parsed data.
+            attachments = [
+                AnymailInboundMessage.construct_attachment(
+                    content_type=attachment["ContentType"],
+                    # Real payloads have "Content", test payloads have "Data" (?!):
+                    content=attachment.get("Content") or attachment["Data"],
+                    base64=True,
+                    filename=attachment.get("Name"),
+                    content_id=attachment.get("ContentID"),
                 )
+                for attachment in esp_event.get("Attachments", [])
+            ]
+            message = AnymailInboundMessage.construct(
+                from_email=self._address(esp_event.get("FromFull")),
+                to=self._addresses(esp_event.get("ToFull")),
+                cc=self._addresses(esp_event.get("CcFull")),
+                bcc=self._addresses(esp_event.get("BccFull")),
+                subject=esp_event.get("Subject", ""),
+                headers=((header["Name"], header["Value"]) for header in headers),
+                text=esp_event.get("TextBody", ""),
+                html=esp_event.get("HtmlBody", ""),
+                attachments=attachments,
+            )
+            # Postmark strips these headers and provides them as separate event fields:
+            if esp_event.get("Date") and "Date" not in message:
+                message["Date"] = esp_event["Date"]
+            if esp_event.get("ReplyTo") and "Reply-To" not in message:
+                message["Reply-To"] = esp_event["ReplyTo"]
 
-        message.envelope_recipient = esp_event.get("OriginalRecipient", None)
-        message.stripped_text = esp_event.get("StrippedTextReply", None)
+        message.envelope_sender = envelope_sender
+        message.envelope_recipient = esp_event.get("OriginalRecipient")
+        message.stripped_text = esp_event.get("StrippedTextReply")
 
         message.spam_detected = message.get("X-Spam-Status", "No").lower() == "yes"
         try:
@@ -249,11 +237,11 @@ def esp_to_anymail_event(self, esp_event):
             message=message,
         )
 
-    @staticmethod
-    def _address(full):
+    @classmethod
+    def _address(cls, full):
         """
         Return a formatted email address
-        from a Postmark inbound {From,To,Cc}Full dict
+        from a Postmark inbound {From,To,Cc,Bcc}Full dict
         """
         if full is None:
             return ""
@@ -263,3 +251,13 @@ def _address(full):
                 addr_spec=full.get("Email", ""),
             )
         )
+
+    @classmethod
+    def _addresses(cls, full_list):
+        """
+        Return a formatted email address list string
+        from a Postmark inbound {To,Cc,Bcc}Full[] list of dicts
+        """
+        if full_list is None:
+            return None
+        return ", ".join(cls._address(addr) for addr in full_list)

docs/esps/postmark.rst

@@ -244,19 +244,28 @@ a `dict` of Postmark `delivery <https://postmarkapp.com/developer/webhooks/deliv
 Inbound webhook
 ---------------
 
-If you want to receive email from Postmark through Anymail's normalized :ref:`inbound <inbound>`
-handling, follow Postmark's `Inbound Processing`_ guide to configure
-an inbound server pointing to Anymail's inbound webhook.
+To receive email from Postmark through Anymail's normalized
+:ref:`inbound <inbound>` handling, follow Postmark's guide to
+`Configure an inbound server`_ that posts to Anymail's inbound webhook.
 
-The InboundHookUrl setting will be:
+In their step 4, set the inbound webhook URL to:
 
    :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/postmark/inbound/`
 
      * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret
      * *yoursite.example.com* is your Django site
 
-Anymail handles the "parse an email" part of Postmark's instructions for you, but you'll
-likely want to work through the other sections to set up a custom inbound domain, and
-perhaps configure inbound spam blocking.
+We recommend enabling the "Include raw email content in JSON payload" checkbox.
+Anymail's inbound handling supports either choice, but raw email is preferred
+to get the most accurate representation of any received message. (If you are using
+Postmark's server API, this is the ``RawEmailEnabled`` option.)
 
+.. versionchanged:: 10.0
+   Added handling for Postmark's "include raw email content".
+
+You may also want to read through the "Inbound domain forwarding" and
+"Configure inbound blocking" sections of Postmark's `Inbound Processing`_ guide.
+
+.. _Configure an inbound server:
+   https://postmarkapp.com/developer/user-guide/inbound/configure-an-inbound-server
 .. _Inbound Processing: https://postmarkapp.com/developer/user-guide/inbound

tests/test_files/postmark-inbound-test-payload-with-raw.json

@@ -0,0 +1,64 @@
+{
+  "FromName": "Postmarkapp Support",
+  "MessageStream": "inbound",
+  "From": "[email protected]",
+  "FromFull": {
+    "Email": "[email protected]",
+    "Name": "Postmarkapp Support",
+    "MailboxHash": ""
+  },
+  "To": "\"Firstname Lastname\" <[email protected]>",
+  "ToFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "Firstname Lastname",
+      "MailboxHash": "SampleHash"
+    }
+  ],
+  "Cc": "\"First Cc\" <[email protected]>, [email protected]",
+  "CcFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "First Cc",
+      "MailboxHash": ""
+    },
+    {
+      "Email": "[email protected]",
+      "Name": "",
+      "MailboxHash": ""
+    }
+  ],
+  "Bcc": "\"First Bcc\" <[email protected]>",
+  "BccFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "First Bcc",
+      "MailboxHash": ""
+    }
+  ],
+  "OriginalRecipient": "[email protected]",
+  "Subject": "Test subject",
+  "MessageID": "00000000-0000-0000-0000-000000000000",
+  "ReplyTo": "[email protected]",
+  "MailboxHash": "SampleHash",
+  "Date": "Fri, 5 May 2023 17:41:16 -0400",
+  "TextBody": "This is a test text body.",
+  "HtmlBody": "<html><body><p>This is a test html body.<\/p><\/body><\/html>",
+  "StrippedTextReply": "This is the reply text",
+  "RawEmail": "From: Postmarkapp Support <[email protected]>\r\nTo: Firstname Lastname <[email protected]>\r\nSubject: Test subject\r\n\r\nThis is a test text body.\r\n",
+  "Tag": "TestTag",
+  "Headers": [
+    {
+      "Name": "X-Header-Test",
+      "Value": ""
+    }
+  ],
+  "Attachments": [
+    {
+      "Name": "test.txt",
+      "ContentType": "text/plain",
+      "Data": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu",
+      "ContentLength": 45
+    }
+  ]
+}

tests/test_files/postmark-inbound-test-payload.json

@@ -0,0 +1,63 @@
+{
+  "FromName": "Postmarkapp Support",
+  "MessageStream": "inbound",
+  "From": "[email protected]",
+  "FromFull": {
+    "Email": "[email protected]",
+    "Name": "Postmarkapp Support",
+    "MailboxHash": ""
+  },
+  "To": "\"Firstname Lastname\" <[email protected]>",
+  "ToFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "Firstname Lastname",
+      "MailboxHash": "SampleHash"
+    }
+  ],
+  "Cc": "\"First Cc\" <[email protected]>, [email protected]",
+  "CcFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "First Cc",
+      "MailboxHash": ""
+    },
+    {
+      "Email": "[email protected]",
+      "Name": "",
+      "MailboxHash": ""
+    }
+  ],
+  "Bcc": "\"First Bcc\" <[email protected]>",
+  "BccFull": [
+    {
+      "Email": "[email protected]",
+      "Name": "First Bcc",
+      "MailboxHash": ""
+    }
+  ],
+  "OriginalRecipient": "[email protected]",
+  "Subject": "Test subject",
+  "MessageID": "00000000-0000-0000-0000-000000000000",
+  "ReplyTo": "[email protected]",
+  "MailboxHash": "SampleHash",
+  "Date": "Fri, 5 May 2023 17:44:33 -0400",
+  "TextBody": "This is a test text body.",
+  "HtmlBody": "<html><body><p>This is a test html body.<\/p><\/body><\/html>",
+  "StrippedTextReply": "This is the reply text",
+  "Tag": "TestTag",
+  "Headers": [
+    {
+      "Name": "X-Header-Test",
+      "Value": ""
+    }
+  ],
+  "Attachments": [
+    {
+      "Name": "test.txt",
+      "ContentType": "text/plain",
+      "Data": "VGhpcyBpcyBhdHRhY2htZW50IGNvbnRlbnRzLCBiYXNlLTY0IGVuY29kZWQu",
+      "ContentLength": 45
+    }
+  ]
+}