Handle Django lazy strings.

In BasePayload, ensure any Django ugettext_lazy
(or similar) are converted to real strings before
handing off to ESP code. This resolves problems where
calling code expects it can use lazy strings "anywhere",
but non-Django code (requests, ESP packages) don't
always handle them correctly.

* Add utils helpers for lazy objects (is_lazy, force_non_lazy*)
* Add lazy object handling to utils.Attachment
* Add lazy object handling converters to BasePayload attr
  processing where appropriate. (This ends up varying by
  the expected attribute type.)

Fixes #34.

Author: medmunds

anymail/backends/base.py

@@ -7,7 +7,8 @@
 from ..exceptions import AnymailCancelSend, AnymailError, AnymailUnsupportedFeature, AnymailRecipientsRefused
 from ..message import AnymailStatus
 from ..signals import pre_send, post_send
-from ..utils import Attachment, ParsedEmail, UNSET, combine, last, get_anymail_setting
+from ..utils import (Attachment, ParsedEmail, UNSET, combine, last, get_anymail_setting,
+                     force_non_lazy, force_non_lazy_list, force_non_lazy_dict)
 
 
 class AnymailBaseBackend(BaseEmailBackend):
@@ -195,31 +196,43 @@ def esp_name(self):
 
 
 class BasePayload(object):
-    # attr, combiner, converter
+    # Listing of EmailMessage/EmailMultiAlternatives attributes
+    # to process into Payload. Each item is in the form:
+    #   (attr, combiner, converter)
+    #   attr: the property name
+    #   combiner: optional function(default_value, value) -> value
+    #     to combine settings defaults with the EmailMessage property value
+    #     (usually `combine` to merge, or `last` for message value to override default;
+    #     use `None` if settings defaults aren't supported)
+    #   converter: optional function(value) -> value transformation
+    #     (can be a callable or the string name of a Payload method, or `None`)
+    #     The converter must force any Django lazy translation strings to text.
+    # The Payload's `set_<attr>` method will be called with
+    # the combined/converted results for each attr.
     base_message_attrs = (
         # Standard EmailMessage/EmailMultiAlternatives props
         ('from_email', last, 'parsed_email'),
         ('to', combine, 'parsed_emails'),
         ('cc', combine, 'parsed_emails'),
         ('bcc', combine, 'parsed_emails'),
-        ('subject', last, None),
+        ('subject', last, force_non_lazy),
         ('reply_to', combine, 'parsed_emails'),
-        ('extra_headers', combine, None),
-        ('body', last, None),  # special handling below checks message.content_subtype
-        ('alternatives', combine, None),
+        ('extra_headers', combine, force_non_lazy_dict),
+        ('body', last, force_non_lazy),  # special handling below checks message.content_subtype
+        ('alternatives', combine, 'prepped_alternatives'),
         ('attachments', combine, 'prepped_attachments'),
     )
     anymail_message_attrs = (
         # Anymail expando-props
-        ('metadata', combine, None),
+        ('metadata', combine, force_non_lazy_dict),
         ('send_at', last, 'aware_datetime'),
-        ('tags', combine, None),
+        ('tags', combine, force_non_lazy_list),
         ('track_clicks', last, None),
         ('track_opens', last, None),
-        ('template_id', last, None),
-        ('merge_data', combine, None),
-        ('merge_global_data', combine, None),
-        ('esp_extra', combine, None),
+        ('template_id', last, force_non_lazy),
+        ('merge_data', combine, force_non_lazy_dict),
+        ('merge_global_data', combine, force_non_lazy_dict),
+        ('esp_extra', combine, force_non_lazy_dict),
     )
     esp_message_attrs = ()  # subclasses can override
 
@@ -261,15 +274,21 @@ def unsupported_feature(self, feature):
     #
 
     def parsed_email(self, address):
-        return ParsedEmail(address, self.message.encoding)
+        return ParsedEmail(address, self.message.encoding)  # (handles lazy address)
 
     def parsed_emails(self, addresses):
         encoding = self.message.encoding
-        return [ParsedEmail(address, encoding) for address in addresses]
+        return [ParsedEmail(address, encoding)  # (handles lazy address)
+                for address in addresses]
+
+    def prepped_alternatives(self, alternatives):
+        return [(force_non_lazy(content), mimetype)
+                for (content, mimetype) in alternatives]
 
     def prepped_attachments(self, attachments):
         str_encoding = self.message.encoding or settings.DEFAULT_CHARSET
-        return [Attachment(attachment, str_encoding) for attachment in attachments]
+        return [Attachment(attachment, str_encoding)  # (handles lazy content, filename)
+                for attachment in attachments]
 
     def aware_datetime(self, value):
         """Converts a date or datetime or timestamp to an aware datetime.

anymail/utils.py

@@ -9,6 +9,7 @@
 from django.conf import settings
 from django.core.mail.message import sanitize_address, DEFAULT_ATTACHMENT_MIME_TYPE
 from django.utils.encoding import force_text
+from django.utils.functional import Promise
 from django.utils.timezone import utc
 
 from .exceptions import AnymailConfigurationError, AnymailInvalidAddress
@@ -162,6 +163,9 @@ def __init__(self, attachment, encoding):
         else:
             (self.name, self.content, self.mimetype) = attachment
 
+        self.name = force_non_lazy(self.name)
+        self.content = force_non_lazy(self.content)
+
         # Guess missing mimetype from filename, borrowed from
         # django.core.mail.EmailMessage._create_attachment()
         if self.mimetype is None and self.name is not None:
@@ -289,3 +293,37 @@ def rfc2822date(dt):
     # but treats naive datetimes as local rather than "UTC with no information ..."
     timeval = timestamp(dt)
     return formatdate(timeval, usegmt=True)
+
+
+def is_lazy(obj):
+    """Return True if obj is a Django lazy object."""
+    # See django.utils.functional.lazy. (This appears to be preferred
+    # to checking for `not isinstance(obj, six.text_type)`.)
+    return isinstance(obj, Promise)
+
+
+def force_non_lazy(obj):
+    """If obj is a Django lazy object, return it coerced to text; otherwise return it unchanged.
+
+    (Similar to django.utils.encoding.force_text, but doesn't alter non-text objects.)
+    """
+    if is_lazy(obj):
+        return six.text_type(obj)
+
+    return obj
+
+
+def force_non_lazy_list(obj):
+    """Return a (shallow) copy of sequence obj, with all values forced non-lazy."""
+    try:
+        return [force_non_lazy(item) for item in obj]
+    except (AttributeError, TypeError):
+        return force_non_lazy(obj)
+
+
+def force_non_lazy_dict(obj):
+    """Return a (deep) copy of dict obj, with all values forced non-lazy."""
+    try:
+        return {key: force_non_lazy_dict(value) for key, value in obj.items()}
+    except (AttributeError, TypeError):
+        return force_non_lazy(obj)

tests/test_general_backend.py

@@ -1,10 +1,14 @@
 from datetime import datetime
+from email.mime.text import MIMEText
 
+import six
 from django.core.exceptions import ImproperlyConfigured
 from django.core.mail import get_connection, send_mail
 from django.test import SimpleTestCase
 from django.test.utils import override_settings
+from django.utils.functional import Promise
 from django.utils.timezone import utc
+from django.utils.translation import ugettext_lazy
 
 from anymail.exceptions import AnymailConfigurationError, AnymailUnsupportedFeature
 from anymail.message import AnymailMessage
@@ -212,3 +216,80 @@ def test_esp_send_defaults_override_globals(self):
         self.assertEqual(params['template_id'], 'global-template')  # global-defaults only
         self.assertEqual(params['espextra'], 'espsetting')
         self.assertNotIn('globalextra', params)  # entire esp_extra is overriden by esp-send-defaults
+
+
+class LazyStringsTest(TestBackendTestCase):
+    """
+    Tests ugettext_lazy strings forced real before passing to ESP transport.
+
+    Docs notwithstanding, Django lazy strings *don't* work anywhere regular
+    strings would. In particular, they aren't instances of unicode/str.
+    There are some cases (e.g., urllib.urlencode, requests' _encode_params)
+    where this can cause encoding errors or just very wrong results.
+
+    Since Anymail sits on the border between Django app code and non-Django
+    ESP code (e.g., requests), it's responsible for converting lazy text
+    to actual strings.
+    """
+
+    def assertNotLazy(self, s, msg=None):
+        self.assertNotIsInstance(s, Promise,
+                                 msg=msg or "String %r is lazy" % six.text_type(s))
+
+    def test_lazy_from(self):
+        # This sometimes ends up lazy when settings.DEFAULT_FROM_EMAIL is meant to be localized
+        self.message.from_email = ugettext_lazy(u'"Global Sales" <[email protected]>')
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['from'].address)
+
+    def test_lazy_subject(self):
+        self.message.subject = ugettext_lazy("subject")
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['subject'])
+
+    def test_lazy_body(self):
+        self.message.body = ugettext_lazy("text body")
+        self.message.attach_alternative(ugettext_lazy("html body"), "text/html")
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['text_body'])
+        self.assertNotLazy(params['html_body'])
+
+    def test_lazy_headers(self):
+        self.message.extra_headers['X-Test'] = ugettext_lazy("Test Header")
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['extra_headers']['X-Test'])
+
+    def test_lazy_attachments(self):
+        self.message.attach(ugettext_lazy("test.csv"), ugettext_lazy("test,csv,data"), "text/csv")
+        self.message.attach(MIMEText(ugettext_lazy("contact info")))
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['attachments'][0].name)
+        self.assertNotLazy(params['attachments'][0].content)
+        self.assertNotLazy(params['attachments'][1].content)
+
+    def test_lazy_tags(self):
+        self.message.tags = [ugettext_lazy("Shipping"), ugettext_lazy("Sales")]
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['tags'][0])
+        self.assertNotLazy(params['tags'][1])
+
+    def test_lazy_metadata(self):
+        self.message.metadata = {'order_type': ugettext_lazy("Subscription")}
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['metadata']['order_type'])
+
+    def test_lazy_merge_data(self):
+        self.message.merge_data = {
+            '[email protected]': {'duration': ugettext_lazy("One Month")}}
+        self.message.merge_global_data = {'order_type': ugettext_lazy("Subscription")}
+        self.message.send()
+        params = self.get_send_params()
+        self.assertNotLazy(params['merge_data']['[email protected]']['duration'])
+        self.assertNotLazy(params['merge_global_data']['order_type'])

tests/test_utils.py

@@ -1,10 +1,11 @@
 # Tests for the anymail/utils.py module
 # (not to be confused with utilities for testing found in in tests/utils.py)
-
+import six
 from django.test import SimpleTestCase
+from django.utils.translation import ugettext_lazy, string_concat
 
 from anymail.exceptions import AnymailInvalidAddress
-from anymail.utils import ParsedEmail
+from anymail.utils import ParsedEmail, is_lazy, force_non_lazy, force_non_lazy_dict, force_non_lazy_list
 
 
 class ParsedEmailTests(SimpleTestCase):
@@ -61,3 +62,57 @@ def test_empty_address(self):
     def test_whitespace_only_address(self):
         with self.assertRaises(AnymailInvalidAddress):
             ParsedEmail(' ', self.ADDRESS_ENCODING)
+
+
+class LazyCoercionTests(SimpleTestCase):
+    """Test utils.is_lazy and force_non_lazy*"""
+
+    def test_is_lazy(self):
+        self.assertTrue(is_lazy(ugettext_lazy("lazy string is lazy")))
+        self.assertTrue(is_lazy(string_concat(ugettext_lazy("concatenation"),
+                                              ugettext_lazy("is lazy"))))
+
+    def test_not_lazy(self):
+        self.assertFalse(is_lazy(u"text not lazy"))
+        self.assertFalse(is_lazy(b"bytes not lazy"))
+        self.assertFalse(is_lazy(None))
+        self.assertFalse(is_lazy({'dict': "not lazy"}))
+        self.assertFalse(is_lazy(["list", "not lazy"]))
+        self.assertFalse(is_lazy(object()))
+        self.assertFalse(is_lazy([ugettext_lazy("doesn't recurse")]))
+
+    def test_force_lazy(self):
+        result = force_non_lazy(ugettext_lazy(u"text"))
+        self.assertIsInstance(result, six.text_type)
+        self.assertEqual(result, u"text")
+
+    def test_force_concat(self):
+        result = force_non_lazy(string_concat(ugettext_lazy(u"text"), ugettext_lazy("concat")))
+        self.assertIsInstance(result, six.text_type)
+        self.assertEqual(result, u"textconcat")
+
+    def test_force_string(self):
+        result = force_non_lazy(u"text")
+        self.assertIsInstance(result, six.text_type)
+        self.assertEqual(result, u"text")
+
+    def test_force_bytes(self):
+        result = force_non_lazy(b"bytes \xFE")
+        self.assertIsInstance(result, six.binary_type)
+        self.assertEqual(result, b"bytes \xFE")
+
+    def test_force_none(self):
+        result = force_non_lazy(None)
+        self.assertIsNone(result)
+
+    def test_force_dict(self):
+        result = force_non_lazy_dict({'a': 1, 'b': ugettext_lazy(u"b"),
+                                 'c': {'c1': ugettext_lazy(u"c1")}})
+        self.assertEqual(result, {'a': 1, 'b': u"b", 'c': {'c1': u"c1"}})
+        self.assertIsInstance(result['b'], six.text_type)
+        self.assertIsInstance(result['c']['c1'], six.text_type)
+
+    def test_force_list(self):
+        result = force_non_lazy_list([0, ugettext_lazy(u"b"), u"c"])
+        self.assertEqual(result, [0, u"b", u"c"])  # coerced to list
+        self.assertIsInstance(result[1], six.text_type)