applying backoff for rate-limited Maven downloads

setup.py

@@ -17,16 +17,20 @@
 
 import os
 import shutil
+from time import sleep
+
 from setuptools import Command
 from setuptools import setup
 from setuptools.command.install import install
 
 if sys.version_info[0] >= 3:
     # Python 3
     from urllib.request import urlopen
+    from urllib.error import HTTPError
 else:
     # Python 2
     from urllib2 import urlopen
+    from urllib2 import HTTPError
 
 #
 # This script modifies the basic setuptools by adding some functionality to the standard
@@ -50,7 +54,7 @@
 
 PACKAGE_NAME = 'amazon_kclpy'
 JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars')
-PACKAGE_VERSION = '2.0.1'
+PACKAGE_VERSION = '2.0.5'
 PYTHON_REQUIREMENTS = [
     'boto',
     # argparse is part of python2.7 but must be declared for python2.6
@@ -121,6 +125,7 @@
     ('commons-beanutils', 'commons-beanutils', '1.9.3'),
     ('commons-collections', 'commons-collections', '3.2.2')
 ]
+MAX_URL_DOWNLOAD_ATTEMPTS = 5
 
 
 class MavenJarDownloader:
@@ -179,9 +184,9 @@ def download_file(self, url, dest):
         """
         Downloads a file at the url to the destination.
         """
-        print('Attempting to retrieve remote jar {url}'.format(url=url))
         try:
-            response = urlopen(url)
+            response = self.make_request_with_backoff(url)
+
             with open(dest, 'wb') as dest_file:
                 shutil.copyfileobj(response, dest_file)
             print('Saving {url} -> {dest}'.format(url=url, dest=dest))
@@ -198,6 +203,21 @@ def download_files(self):
                 url = self.package_url(package[0], package[1], package[2])
                 self.download_file(url, dest)
 
+    def make_request_with_backoff(self, url):
+        for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS):
+            print('Attempting to retrieve remote jar {url}'.format(url=url))
+            try:
+                return urlopen(url)
+            except HTTPError as e:
+                if e.code == 429:
+                    sleep_time = 2 ** attempt_number
+                    print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time))
+                    sleep(sleep_time)
+                else:
+                    raise
+
+        raise Exception('"429 Too Many Requests" responses received.')
+
 
 class DownloadJarsCommand(Command):
     description = "Download the jar files needed to run the sample application"