Workaround missing smtp-id in SendGrid tracking.

* Add smtp-id in unique_args (metadata), to ensure
  it shows up in click and open events.
* Add SENDGRID_GENERATE_MESSAGE_ID setting,
  default True, to control auto-Message-ID behavior.
* Document it.

Author: medmunds

anymail/backends/sendgrid.py

@@ -1,3 +1,5 @@
+from email.utils import unquote
+
 from django.core.exceptions import ImproperlyConfigured
 from django.core.mail import make_msgid
 from requests.structures import CaseInsensitiveDict
@@ -9,7 +11,6 @@
 from .base_requests import AnymailRequestsBackend, RequestsPayload
 
 
-
 class SendGridBackend(AnymailRequestsBackend):
     """
     SendGrid API Email Backend
@@ -27,6 +28,8 @@ def __init__(self, **kwargs):
                 "SENDGRID_PASSWORD in your Django ANYMAIL settings."
             )
 
+        self.generate_message_id = get_anymail_setting('SENDGRID_GENERATE_MESSAGE_ID', default=True)
+
         # This is SendGrid's Web API v2 (because the Web API v3 doesn't support sending)
         api_url = get_anymail_setting("SENDGRID_API_URL", "https://api.sendgrid.com/api/")
         if not api_url.endswith("/"):
@@ -56,6 +59,7 @@ class SendGridPayload(RequestsPayload):
 
     def __init__(self, message, defaults, backend, *args, **kwargs):
         self.all_recipients = []  # used for backend.parse_recipient_status
+        self.generate_message_id = backend.generate_message_id
         self.message_id = None  # Message-ID -- assigned in serialize_data unless provided in headers
         self.smtpapi = {}  # SendGrid x-smtpapi field
 
@@ -76,6 +80,9 @@ def get_api_endpoint(self):
     def serialize_data(self):
         """Performs any necessary serialization on self.data, and returns the result."""
 
+        if self.generate_message_id:
+            self.ensure_message_id()
+
         # Serialize x-smtpapi to json:
         if len(self.smtpapi) > 0:
             # If esp_extra was also used to set x-smtpapi, need to merge it
@@ -86,16 +93,26 @@ def serialize_data(self):
         elif "x-smtpapi" in self.data:
             self.data["x-smtpapi"] = self.serialize_json(self.data["x-smtpapi"])
 
-        # Add our own message_id, and serialize extra headers to json:
+        # Serialize extra headers to json:
         headers = self.data["headers"]
-        try:
-            self.message_id = headers["Message-ID"]
-        except KeyError:
-            self.message_id = headers["Message-ID"] = self.make_message_id()
         self.data["headers"] = self.serialize_json(dict(headers.items()))
 
         return self.data
 
+    def ensure_message_id(self):
+        """Ensure message has a known Message-ID for later event tracking"""
+        headers = self.data["headers"]
+        if "Message-ID" not in headers:
+            # Only make our own if caller hasn't already provided one
+            headers["Message-ID"] = self.make_message_id()
+        self.message_id = headers["Message-ID"]
+
+        # Workaround for missing message ID (smtp-id) in SendGrid engagement events
+        # (click and open tracking): because unique_args get merged into the raw event
+        # record, we can supply the 'smtp-id' field for any events missing it.
+        # Must use the unquoted (no <angle brackets>) version to match other SendGrid APIs.
+        self.smtpapi.setdefault('unique_args', {})['smtp-id'] = unquote(self.message_id)
+
     def make_message_id(self):
         """Returns a Message-ID that could be used for this payload
 

docs/esps/sendgrid.rst

@@ -78,6 +78,17 @@ nor ``ANYMAIL_SENDGRID_USERNAME`` is set.
 .. _SendGrid credentials settings: https://app.sendgrid.com/settings/credentials
 
 
+.. setting:: ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID
+
+.. rubric:: SENDGRID_GENERATE_MESSAGE_ID
+
+Whether Anymail should generate a Message-ID for messages sent
+through SendGrid, to facilitate event tracking.
+
+Default ``True``. You can set to ``False`` to disable this behavior.
+See :ref:`Message-ID quirks <sendgrid-message-id>` below.
+
+
 .. setting:: ANYMAIL_SENDGRID_API_URL
 
 .. rubric:: SENDGRID_API_URL
@@ -132,23 +143,30 @@ Limitations and quirks
   make sure each one has a unique, non-empty filename.
 
 
+.. _sendgrid-message-id:
+
 **Message-ID**
   SendGrid does not return any sort of unique id from its send API call.
   Knowing a sent message's ID can be important for later queries about
   the message's status.
 
-  To work around this, Anymail generates a new Message-ID for each
+  To work around this, Anymail by default generates a new Message-ID for each
   outgoing message, provides it to SendGrid, and includes it in the
   :attr:`~anymail.message.AnymailMessage.anymail_status`
   attribute after you send the message.
 
   In later SendGrid API calls, you can match that Message-ID
-  to SendGrid's ``smtp-id`` event field.
+  to SendGrid's ``smtp-id`` event field. (Anymail uses an additional
+  workaround to ensure smtp-id is included in all SendGrid events,
+  even those that aren't documented to include it.)
 
   Anymail will use the domain of the message's :attr:`from_email`
   to generate the Message-ID. (If this isn't desired, you can supply
   your own Message-ID in the message's :attr:`extra_headers`.)
 
+  To disable all of these Message-ID workarounds, set
+  :setting:`ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID` to False in your settings.
+
 
 **Invalid Addresses**
   SendGrid will accept *and send* just about anything as

tests/test_sendgrid_backend.py

@@ -64,6 +64,9 @@ def test_send_mail(self):
         # make sure backend assigned a Message-ID for event tracking
         email_headers = json.loads(data['headers'])
         self.assertRegex(email_headers['Message-ID'], r'\<.+@sender\.example\.com\>')  # id uses from_email's domain
+        # make sure we added the unquoted Message-ID to unique_args for event notification
+        smtpapi = self.get_smtpapi()
+        self.assertEqual(email_headers['Message-ID'], '<{}>'.format(smtpapi['unique_args']['smtp-id']))
 
     @override_settings(ANYMAIL={'SENDGRID_USERNAME': 'sg_username', 'SENDGRID_PASSWORD': 'sg_password'})
     def test_user_pass_auth(self):
@@ -124,6 +127,10 @@ def test_email_message(self):
             'Reply-To': '[email protected]',
             'X-MyHeader': 'my value',
         })
+        # make sure custom Message-ID also added to unique_args
+        self.assertJSONEqual(data['x-smtpapi'], {
+            'unique_args': {'smtp-id': '[email protected]'}
+        })
 
     def test_html_message(self):
         text_content = 'This is an important message.'
@@ -347,6 +354,7 @@ def test_metadata(self):
         self.message.metadata = {'user_id': "12345", 'items': 6}
         self.message.send()
         smtpapi = self.get_smtpapi()
+        smtpapi['unique_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(smtpapi['unique_args'], {'user_id': "12345", 'items': 6})
 
     def test_send_at(self):
@@ -401,6 +409,7 @@ def test_tracking(self):
         self.assertEqual(smtpapi['filters']['clicktrack'], {'settings': {'enable': 1}})
         self.assertEqual(smtpapi['filters']['opentrack'], {'settings': {'enable': 0}})
 
+    @override_settings(ANYMAIL_SENDGRID_GENERATE_MESSAGE_ID=False)  # else we force unique_args
     def test_default_omits_options(self):
         """Make sure by default we don't send any ESP-specific options.
 
@@ -508,6 +517,7 @@ def test_send_defaults(self):
         data = self.get_api_call_data()
         smtpapi = self.get_smtpapi()
         # All these values came from ANYMAIL_SEND_DEFAULTS:
+        smtpapi['unique_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(smtpapi['unique_args'], {'global': 'globalvalue', 'other': 'othervalue'})
         self.assertEqual(smtpapi['category'], ['globaltag'])
         self.assertEqual(smtpapi['filters']['clicktrack']['settings']['enable'], 1)
@@ -525,6 +535,7 @@ def test_merge_message_with_send_defaults(self):
         data = self.get_api_call_data()
         smtpapi = self.get_smtpapi()
         # All these values came from ANYMAIL_SEND_DEFAULTS + message.*:
+        smtpapi['unique_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(smtpapi['unique_args'], {
             'global': 'globalvalue',
             'message': 'messagevalue',  # additional metadata
@@ -547,6 +558,7 @@ def test_esp_send_defaults(self):
         data = self.get_api_call_data()
         smtpapi = self.get_smtpapi()
         # All these values came from ANYMAIL_SEND_DEFAULTS plus ANYMAIL_SENDGRID_SEND_DEFAULTS:
+        smtpapi['unique_args'].pop('smtp-id', None)  # remove Message-ID we added as tracking workaround
         self.assertEqual(smtpapi['unique_args'], {'esp': 'espvalue'})  # entire metadata overridden
         self.assertCountEqual(smtpapi['category'], ['esptag'])  # entire tags overridden
         self.assertEqual(smtpapi['filters']['clicktrack']['settings']['enable'], 1)  # no override