Tests: limit live API integration tests in Travis runs

To conserve our ESP test accounts' send quotas, don't run
the live API integration tests 13 times in every Travis run.
Instead, just run them twice, on a representative set
of Python/Django combinations:

* Once on Python 2.7 (currently with Django 1.8)
* Once on Python 3.x (currently 3.5 with Django 1.9)

(Prep for running weekly tests on Travis cron.)

The *non*-integration tests still run on all combos.

* Introduce RUN_LIVE_TESTS environment var to control
  whether live API integration test cases should run.
  Default True, except in Travis-CI runs default False.
* Enable RUN_LIVE_TESTS in .travis.yml matrix for the
  Python/Django combos listed above.

(cherry picked from commit 0c5911c)

Author: medmunds

.travis.yml

@@ -4,16 +4,20 @@ matrix:
   include:
     # Anymail supports the same python versions as Django, excluding Python 3.2, but adding pypy.
     # https://docs.djangoproject.com/en/dev/faq/install/#what-python-version-can-i-use-with-django
+
+    # Live API integration tests are only run on a few, representative Python/Django version
+    # cominbations, to avoid rapidly consuming the testing accounts' entire send allotments.
+
     # Django 1.8: Python 2.7, 3.2 (until the end of 2016), 3.3, 3.4, 3.5
-    - { env: DJANGO=django==1.8, python: 2.7 }
+    - { env: DJANGO=django==1.8 RUN_LIVE_TESTS=true, python: 2.7 }
     - { env: DJANGO=django==1.8, python: 3.3 }
     - { env: DJANGO=django==1.8, python: 3.4 }
     - { env: DJANGO=django==1.8, python: 3.5 }
     - { env: DJANGO=django==1.8, python: pypy }
     # Django 1.9: Python 2.7, 3.4, 3.5
     - { env: DJANGO=django==1.9, python: 2.7 }
     - { env: DJANGO=django==1.9, python: 3.4 }
-    - { env: DJANGO=django==1.9, python: 3.5 }
+    - { env: DJANGO=django==1.9 RUN_LIVE_TESTS=true, python: 3.5 }
     - { env: DJANGO=django==1.9, python: pypy }
     # Django 1.10 (prerelease): Python 2.7, 3.4, 3.5
     - { env: DJANGO="--pre django", python: 2.7 }

tests/test_mailgun_integration.py

@@ -13,12 +13,13 @@
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path
+from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
 
 MAILGUN_TEST_API_KEY = os.getenv('MAILGUN_TEST_API_KEY')
 MAILGUN_TEST_DOMAIN = os.getenv('MAILGUN_TEST_DOMAIN')
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN,
                      "Set MAILGUN_TEST_API_KEY and MAILGUN_TEST_DOMAIN environment variables "
                      "to run Mailgun integration tests")

tests/test_mandrill_integration.py

@@ -10,11 +10,12 @@
 from anymail.exceptions import AnymailAPIError, AnymailRecipientsRefused
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path
+from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
 
 MANDRILL_TEST_API_KEY = os.getenv('MANDRILL_TEST_API_KEY')
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(MANDRILL_TEST_API_KEY,
                      "Set MANDRILL_TEST_API_KEY environment variable to run integration tests")
 @override_settings(MANDRILL_API_KEY=MANDRILL_TEST_API_KEY,

tests/test_postmark_integration.py

@@ -1,14 +1,17 @@
 from __future__ import unicode_literals
 
+import unittest
+
 from django.test import SimpleTestCase
 from django.test.utils import override_settings
 
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path
+from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @override_settings(ANYMAIL_POSTMARK_SERVER_TOKEN="POSTMARK_API_TEST",
                    EMAIL_BACKEND="anymail.backends.postmark.PostmarkBackend")
 class PostmarkBackendIntegrationTests(SimpleTestCase, AnymailTestMixin):

tests/test_sendgrid_integration.py

@@ -11,7 +11,7 @@
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path
+from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
 
 # For API_KEY auth tests:
 SENDGRID_TEST_API_KEY = os.getenv('SENDGRID_TEST_API_KEY')
@@ -21,6 +21,7 @@
 SENDGRID_TEST_PASSWORD = os.getenv('SENDGRID_TEST_PASSWORD')
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(SENDGRID_TEST_API_KEY,
                      "Set SENDGRID_TEST_API_KEY environment variable "
                      "to run SendGrid integration tests")
@@ -118,6 +119,7 @@ def test_invalid_api_key(self):
         self.assertIn("authorization grant is invalid", str(err))
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(SENDGRID_TEST_USERNAME and SENDGRID_TEST_PASSWORD,
                      "Set SENDGRID_TEST_USERNAME and SENDGRID_TEST_PASSWORD"
                      "environment variables to run SendGrid integration tests")

tests/test_sparkpost_integration.py

@@ -10,11 +10,12 @@
 from anymail.exceptions import AnymailAPIError
 from anymail.message import AnymailMessage
 
-from .utils import AnymailTestMixin, sample_image_path
+from .utils import AnymailTestMixin, sample_image_path, RUN_LIVE_TESTS
 
 SPARKPOST_TEST_API_KEY = os.getenv('SPARKPOST_TEST_API_KEY')
 
 
+@unittest.skipUnless(RUN_LIVE_TESTS, "RUN_LIVE_TESTS disabled in this environment")
 @unittest.skipUnless(SPARKPOST_TEST_API_KEY,
                      "Set SPARKPOST_TEST_API_KEY environment variable "
                      "to run SparkPost integration tests")

tests/utils.py

@@ -1,5 +1,6 @@
 # Anymail test utils
 import sys
+from distutils.util import strtobool
 
 import os
 import re
@@ -10,6 +11,25 @@
 from django.test import Client
 
 
+def envbool(var, default=False):
+    """Returns value of environment variable var as a bool, or default if not set.
+
+    Converts `'true'` to `True`, and `'false'` to `False`.
+    See :func:`~distutils.util.strtobool` for full list of allowable values.
+    """
+    val = os.getenv(var, None)
+    if val is None:
+        return default
+    else:
+        return strtobool(val)
+
+
+# RUN_LIVE_TESTS: whether to run live API integration tests.
+# True by default, except in CONTINUOUS_INTEGRATION job.
+# (See comments and overrides in .travis.yml.)
+RUN_LIVE_TESTS = envbool('RUN_LIVE_TESTS', default=not envbool('CONTINUOUS_INTEGRATION'))
+
+
 def decode_att(att):
     """Returns the original data from base64-encoded attachment content"""
     return b64decode(att.encode('ascii'))