Add merge_metadata for other ESPs

Support merge_metadata in Mailgun, Mailjet, Mandrill, Postmark, 
SparkPost, and Test backends. (SendGrid covered in earlier PR.)

Also:
* Add `merge_metadata` to AnymailMessage, AnymailMessageMixin
* Add `is_batch()` logic to BasePayload, for consistent handling
* Docs

Note: Mailjet implementation switches *all* batch sending from their 
"Recipients" field to to the "Messages" array bulk sending option.
This allows an independent payload for each batch recipient.
In addition to supporting merge_metadata, this also removes the
prior limitation on mixing Cc/Bcc with merge_data.

Closes #141.

Author: medmunds

CHANGELOG.rst

@@ -39,6 +39,17 @@ Breaking changes
   code is doing something like `message.anymail_status.recipients[email.lower()]`,
   you should remove the `.lower()`
 
+Features
+~~~~~~~~
+
+* Add new `merge_metadata` option for providing per-recipient metadata in batch
+  sends. Available for all supported ESPs *except* Amazon SES and SendinBlue.
+  See `docs <https://anymail.readthedocs.io/en/latest/sending/anymail_additions/#anymail.message.AnymailMessage.merge_metadata>`_.
+  (Thanks `@janneThoft`_ for the idea and SendGrid implementation.)
+
+* **Mailjet:** Remove limitation on using `cc` or `bcc` together with `merge_data`.
+
+
 Fixes
 ~~~~~
 
@@ -908,6 +919,7 @@ Features
 .. _@calvin: https://github.com/calvin
 .. _@costela: https://github.com/costela
 .. _@decibyte: https://github.com/decibyte
+.. _@janneThoft: https://github.com/janneThoft
 .. _@joshkersey: https://github.com/joshkersey
 .. _@Lekensteyn: https://github.com/Lekensteyn
 .. _@lewistaylor: https://github.com/lewistaylor

anymail/backends/base.py

@@ -250,11 +250,16 @@ class BasePayload(object):
     )
     esp_message_attrs = ()  # subclasses can override
 
+    # If any of these attrs are set on a message, treat the message
+    # as a batch send (separate message for each `to` recipient):
+    batch_attrs = ('merge_data', 'merge_metadata')
+
     def __init__(self, message, defaults, backend):
         self.message = message
         self.defaults = defaults
         self.backend = backend
         self.esp_name = backend.esp_name
+        self._batch_attrs_used = {attr: UNSET for attr in self.batch_attrs}
 
         self.init_payload()
 
@@ -287,6 +292,20 @@ def __init__(self, message, defaults, backend):
                     # AttributeError here? Your Payload subclass is missing a set_<attr> implementation
                     setter = getattr(self, 'set_%s' % attr)
                 setter(value)
+            if attr in self.batch_attrs:
+                self._batch_attrs_used[attr] = (value is not UNSET)
+
+    def is_batch(self):
+        """
+        Return True if the message should be treated as a batch send.
+
+        Intended to be used inside serialize_data or similar, after all relevant
+        attributes have been processed. Will error if called before that (e.g.,
+        inside a set_<attr> method or during __init__).
+        """
+        batch_attrs_used = self._batch_attrs_used.values()
+        assert UNSET not in batch_attrs_used, "Cannot call is_batch before all attributes processed"
+        return any(batch_attrs_used)
 
     def unsupported_feature(self, feature):
         if not self.backend.ignore_unsupported_features:

anymail/backends/mailgun.py

@@ -68,8 +68,10 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.all_recipients = []  # used for backend.parse_recipient_status
 
         # late-binding of recipient-variables:
-        self.merge_data = None
-        self.merge_global_data = None
+        self.merge_data = {}
+        self.merge_global_data = {}
+        self.metadata = {}
+        self.merge_metadata = {}
         self.to_emails = []
 
         super(MailgunPayload, self).__init__(message, defaults, backend, auth=auth, *args, **kwargs)
@@ -117,32 +119,51 @@ def get_request_params(self, api_url):
         return params
 
     def serialize_data(self):
-        self.populate_recipient_variables()
+        if self.is_batch() or self.merge_global_data:
+            self.populate_recipient_variables()
         return self.data
 
     def populate_recipient_variables(self):
-        """Populate Mailgun recipient-variables header from merge data"""
-        merge_data = self.merge_data
-
-        if self.merge_global_data is not None:
-            # Mailgun doesn't support global variables.
-            # We emulate them by populating recipient-variables for all recipients.
-            if merge_data is not None:
-                merge_data = merge_data.copy()  # don't modify the original, which doesn't belong to us
-            else:
-                merge_data = {}
-            for email in self.to_emails:
-                try:
-                    recipient_data = merge_data[email]
-                except KeyError:
-                    merge_data[email] = self.merge_global_data
-                else:
-                    # Merge globals (recipient_data wins in conflict)
-                    merge_data[email] = self.merge_global_data.copy()
-                    merge_data[email].update(recipient_data)
-
-        if merge_data is not None:
-            self.data['recipient-variables'] = self.serialize_json(merge_data)
+        """Populate Mailgun recipient-variables from merge data and metadata"""
+        merge_metadata_keys = set()  # all keys used in any recipient's merge_metadata
+        for recipient_metadata in self.merge_metadata.values():
+            merge_metadata_keys.update(recipient_metadata.keys())
+        metadata_vars = {key: "v:%s" % key for key in merge_metadata_keys}  # custom-var for key
+
+        # Set up custom-var substitutions for merge metadata
+        # data['v:SomeMergeMetadataKey'] = '%recipient.v:SomeMergeMetadataKey%'
+        for var in metadata_vars.values():
+            self.data[var] = "%recipient.{var}%".format(var=var)
+
+        # Any (toplevel) metadata that is also in (any) merge_metadata must be be moved
+        # into recipient-variables; and all merge_metadata vars must have defaults
+        # (else they'll get the '%recipient.v:SomeMergeMetadataKey%' literal string).
+        base_metadata = {metadata_vars[key]: self.metadata.get(key, '')
+                         for key in merge_metadata_keys}
+
+        recipient_vars = {}
+        for addr in self.to_emails:
+            # For each recipient, Mailgun recipient-variables[addr] is merger of:
+            # 1. metadata, for any keys that appear in merge_metadata
+            recipient_data = base_metadata.copy()
+
+            # 2. merge_metadata[addr], with keys prefixed with 'v:'
+            if addr in self.merge_metadata:
+                recipient_data.update({
+                    metadata_vars[key]: value for key, value in self.merge_metadata[addr].items()
+                })
+
+            # 3. merge_global_data (because Mailgun doesn't support global variables)
+            recipient_data.update(self.merge_global_data)
+
+            # 4. merge_data[addr]
+            if addr in self.merge_data:
+                recipient_data.update(self.merge_data[addr])
+
+            if recipient_data:
+                recipient_vars[addr] = recipient_data
+
+        self.data['recipient-variables'] = self.serialize_json(recipient_vars)
 
     #
     # Payload construction
@@ -210,6 +231,7 @@ def set_envelope_sender(self, email):
         self.sender_domain = email.domain
 
     def set_metadata(self, metadata):
+        self.metadata = metadata  # save for handling merge_metadata later
         for key, value in metadata.items():
             self.data["v:%s" % key] = value
 
@@ -242,6 +264,10 @@ def set_merge_global_data(self, merge_global_data):
         # Processed at serialization time (to allow merging global data)
         self.merge_global_data = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # Processed at serialization time (to allow combining with merge_data)
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Allow override of sender_domain via esp_extra

anymail/backends/mailjet.py

@@ -80,31 +80,48 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
             'Content-Type': 'application/json',
         }
         # Late binding of recipients and their variables
-        self.recipients = {}
-        self.merge_data = None
+        self.recipients = {'to': []}
+        self.metadata = None
+        self.merge_data = {}
+        self.merge_metadata = {}
         super(MailjetPayload, self).__init__(message, defaults, backend,
                                              auth=auth, headers=http_headers, *args, **kwargs)
 
     def get_api_endpoint(self):
         return "send"
 
     def serialize_data(self):
-        self._finish_recipients()
         self._populate_sender_from_template()
+        if self.is_batch():
+            self.data = {'Messages': [
+                self._data_for_recipient(to_addr)
+                for to_addr in self.recipients['to']
+            ]}
         return self.serialize_json(self.data)
 
-    #
-    # Payload construction
-    #
-
-    def _finish_recipients(self):
-        # NOTE do not set both To and Recipients, it behaves specially: each
-        # recipient receives a separate mail but the To address receives one
-        # listing all recipients.
-        if "cc" in self.recipients or "bcc" in self.recipients:
-            self._finish_recipients_single()
-        else:
-            self._finish_recipients_with_vars()
+    def _data_for_recipient(self, email):
+        # Return send data for single recipient, without modifying self.data
+        data = self.data.copy()
+        data['To'] = self._format_email_for_mailjet(email)
+
+        if email.addr_spec in self.merge_data:
+            recipient_merge_data = self.merge_data[email.addr_spec]
+            if 'Vars' in data:
+                data['Vars'] = data['Vars'].copy()  # clone merge_global_data
+                data['Vars'].update(recipient_merge_data)
+            else:
+                data['Vars'] = recipient_merge_data
+
+        if email.addr_spec in self.merge_metadata:
+            recipient_metadata = self.merge_metadata[email.addr_spec]
+            if self.metadata:
+                metadata = self.metadata.copy()  # clone toplevel metadata
+                metadata.update(recipient_metadata)
+            else:
+                metadata = recipient_metadata
+            data["Mj-EventPayLoad"] = self.serialize_json(metadata)
+
+        return data
 
     def _populate_sender_from_template(self):
         # If no From address was given, use the address from the template.
@@ -137,42 +154,21 @@ def _populate_sender_from_template(self):
                                               email_message=self.message, response=response, backend=self.backend)
             self.set_from_email(parsed)
 
-    def _finish_recipients_with_vars(self):
-        """Send bulk mail with different variables for each mail."""
-        assert "Cc" not in self.data and "Bcc" not in self.data
-        recipients = []
-        merge_data = self.merge_data or {}
-        for email in self.recipients["to"]:
-            recipient = {
-                "Email": email.addr_spec,
-                "Name": email.display_name,
-                "Vars": merge_data.get(email.addr_spec)
-            }
-            # Strip out empty Name and Vars
-            recipient = {k: v for k, v in recipient.items() if v}
-            recipients.append(recipient)
-        self.data["Recipients"] = recipients
-
-    def _finish_recipients_single(self):
-        """Send a single mail with some To, Cc and Bcc headers."""
-        assert "Recipients" not in self.data
-        if self.merge_data:
-            # When Cc and Bcc headers are given, then merge data cannot be set.
-            raise NotImplementedError("Cannot set merge data with bcc/cc")
-        for recipient_type, emails in self.recipients.items():
-            # Workaround Mailjet 3.0 bug parsing display-name with commas
-            # (see test_comma_in_display_name in test_mailjet_backend for details)
-            formatted_emails = [
-                email.address if "," not in email.display_name
-                # else name has a comma, so force it into MIME encoded-word utf-8 syntax:
-                else EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8')
-                for email in emails
-            ]
-            self.data[recipient_type.capitalize()] = ", ".join(formatted_emails)
+    def _format_email_for_mailjet(self, email):
+        """Return EmailAddress email converted to a string that Mailjet can parse properly"""
+        # Workaround Mailjet 3.0 bug parsing display-name with commas
+        # (see test_comma_in_display_name in test_mailjet_backend for details)
+        if "," in email.display_name:
+            return EmailAddress(email.display_name.encode('utf-8'), email.addr_spec).formataddr('utf-8')
+        else:
+            return email.address
+
+    #
+    # Payload construction
+    #
 
     def init_payload(self):
-        self.data = {
-        }
+        self.data = {}
 
     def set_from_email(self, email):
         self.data["FromEmail"] = email.addr_spec
@@ -181,9 +177,10 @@ def set_from_email(self, email):
 
     def set_recipients(self, recipient_type, emails):
         assert recipient_type in ["to", "cc", "bcc"]
-        # Will be handled later in serialize_data
         if emails:
-            self.recipients[recipient_type] = emails
+            self.recipients[recipient_type] = emails  # save for recipient_status processing
+            self.data[recipient_type.capitalize()] = ", ".join(
+                [self._format_email_for_mailjet(email) for email in emails])
 
     def set_subject(self, subject):
         self.data["Subject"] = subject
@@ -225,8 +222,8 @@ def set_envelope_sender(self, email):
         self.data["Sender"] = email.addr_spec  # ??? v3 docs unclear
 
     def set_metadata(self, metadata):
-        # Mailjet expects a single string payload
         self.data["Mj-EventPayLoad"] = self.serialize_json(metadata)
+        self.metadata = metadata  # keep original in case we need to merge with merge_metadata
 
     def set_tags(self, tags):
         # The choices here are CustomID or Campaign, and Campaign seems closer
@@ -257,5 +254,9 @@ def set_merge_data(self, merge_data):
     def set_merge_global_data(self, merge_global_data):
         self.data["Vars"] = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # Will be handled later in serialize_data
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)

anymail/backends/mandrill.py

@@ -79,6 +79,9 @@ def get_api_endpoint(self):
 
     def serialize_data(self):
         self.process_esp_extra()
+        if self.is_batch():
+            # hide recipients from each other
+            self.data['message']['preserve_recipients'] = False
         return self.serialize_json(self.data)
 
     #
@@ -163,7 +166,6 @@ def set_template_id(self, template_id):
         self.data.setdefault("template_content", [])  # Mandrill requires something here
 
     def set_merge_data(self, merge_data):
-        self.data['message']['preserve_recipients'] = False  # if merge, hide recipients from each other
         self.data['message']['merge_vars'] = [
             {'rcpt': rcpt, 'vars': [{'name': key, 'content': rcpt_data[key]}
                                     for key in sorted(rcpt_data.keys())]}  # sort for testing reproducibility
@@ -176,6 +178,13 @@ def set_merge_global_data(self, merge_global_data):
             for var, value in merge_global_data.items()
         ]
 
+    def set_merge_metadata(self, merge_metadata):
+        # recipient_metadata format is similar to, but not quite the same as, merge_vars:
+        self.data['message']['recipient_metadata'] = [
+            {'rcpt': rcpt, 'values': rcpt_data}
+            for rcpt, rcpt_data in merge_metadata.items()
+        ]
+
     def set_esp_extra(self, extra):
         # late bind in serialize_data, so that obsolete Djrill attrs can contribute
         self.esp_extra = extra

anymail/backends/postmark.py

@@ -156,10 +156,11 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.to_emails = []
         self.cc_and_bcc_emails = []  # need to track (separately) for parse_recipient_status
         self.merge_data = None
+        self.merge_metadata = None
         super(PostmarkPayload, self).__init__(message, defaults, backend, headers=headers, *args, **kwargs)
 
     def get_api_endpoint(self):
-        batch_send = self.merge_data is not None and len(self.to_emails) > 1
+        batch_send = self.is_batch() and len(self.to_emails) > 1
         if 'TemplateAlias' in self.data or 'TemplateId' in self.data or 'TemplateModel' in self.data:
             if batch_send:
                 return "email/batchWithTemplates"
@@ -197,6 +198,14 @@ def data_for_recipient(self, to):
                 data["TemplateModel"].update(recipient_data)
             else:
                 data["TemplateModel"] = recipient_data
+        if self.merge_metadata and to.addr_spec in self.merge_metadata:
+            recipient_metadata = self.merge_metadata[to.addr_spec]
+            if "Metadata" in data:
+                # merge recipient_metadata into toplevel metadata
+                data["Metadata"] = data["Metadata"].copy()
+                data["Metadata"].update(recipient_metadata)
+            else:
+                data["Metadata"] = recipient_metadata
         return data
 
     #
@@ -298,6 +307,10 @@ def set_merge_data(self, merge_data):
     def set_merge_global_data(self, merge_global_data):
         self.data["TemplateModel"] = merge_global_data
 
+    def set_merge_metadata(self, merge_metadata):
+        # late-bind
+        self.merge_metadata = merge_metadata
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Special handling for 'server_token':