diff --git a/setup.py b/setup.py index 2c047bf..0694723 100644 --- a/setup.py +++ b/setup.py @@ -3,10 +3,11 @@ from __future__ import print_function import glob -import sys - import os import shutil +import sys +import time + from setuptools import Command from setuptools import setup from setuptools.command.install import install @@ -14,9 +15,11 @@ if sys.version_info[0] >= 3: # Python 3 from urllib.request import urlopen + from urllib.error import HTTPError else: # Python 2 from urllib2 import urlopen + from urllib2 import HTTPError # # This script modifies the basic setuptools by adding some functionality to the standard @@ -40,7 +43,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.2' +PACKAGE_VERSION = '2.0.3' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -115,6 +118,7 @@ ('commons-collections', 'commons-collections', '3.2.2'), ('software.amazon.glue', 'schema-registry-serde', '1.0.2') ] +MAX_URL_DOWNLOAD_ATTEMPTS = 5 class MavenJarDownloader: @@ -175,7 +179,8 @@ def download_file(self, url, dest): """ print('Attempting to retrieve remote jar {url}'.format(url=url)) try: - response = urlopen(url) + response = self.make_request_with_backoff(url) + with open(dest, 'wb') as dest_file: shutil.copyfileobj(response, dest_file) print('Saving {url} -> {dest}'.format(url=url, dest=dest)) @@ -192,6 +197,21 @@ def download_files(self): url = self.package_url(package[0], package[1], package[2]) self.download_file(url, dest) + def make_request_with_backoff(self, url): + for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): + print('Attempting to retrieve remote jar {url}'.format(url=url)) + try: + return urlopen(url) + except HTTPError as e: + if e.code == 429: + sleep_time = 2 ** attempt_number + print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) + time.sleep(sleep_time) + else: + raise + + raise Exception('"429 Too Many Requests" responses received.') + class DownloadJarsCommand(Command): description = "Download the jar files needed to run the sample application"