diff --git a/setup.py b/setup.py index 99aba8b..67ef305 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,8 @@ import os import shutil +from time import sleep + from setuptools import Command from setuptools import setup from setuptools.command.install import install @@ -24,9 +26,11 @@ if sys.version_info[0] >= 3: # Python 3 from urllib.request import urlopen + from urllib.error import HTTPError else: # Python 2 from urllib2 import urlopen + from urllib2 import HTTPError # # This script modifies the basic setuptools by adding some functionality to the standard @@ -50,7 +54,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.1' +PACKAGE_VERSION = '2.0.5' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -121,6 +125,7 @@ ('commons-beanutils', 'commons-beanutils', '1.9.3'), ('commons-collections', 'commons-collections', '3.2.2') ] +MAX_URL_DOWNLOAD_ATTEMPTS = 5 class MavenJarDownloader: @@ -179,9 +184,9 @@ def download_file(self, url, dest): """ Downloads a file at the url to the destination. """ - print('Attempting to retrieve remote jar {url}'.format(url=url)) try: - response = urlopen(url) + response = self.make_request_with_backoff(url) + with open(dest, 'wb') as dest_file: shutil.copyfileobj(response, dest_file) print('Saving {url} -> {dest}'.format(url=url, dest=dest)) @@ -198,6 +203,21 @@ def download_files(self): url = self.package_url(package[0], package[1], package[2]) self.download_file(url, dest) + def make_request_with_backoff(self, url): + for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): + print('Attempting to retrieve remote jar {url}'.format(url=url)) + try: + return urlopen(url) + except HTTPError as e: + if e.code == 429: + sleep_time = 2 ** attempt_number + print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) + sleep(sleep_time) + else: + raise + + raise Exception('"429 Too Many Requests" responses received.') + class DownloadJarsCommand(Command): description = "Download the jar files needed to run the sample application"