Merge pull request #48 from vsoch/master

updating to include retrying in google functions with exponential backoff

Author: vsoch

MANIFEST.in

@@ -1,3 +1,5 @@
+include README.md
+include LICENSE
 recursive-include singularity/templates *
 recursive-include singularity/static *
 recursive-include singularity/build *

requirements.txt

@@ -1,11 +1,12 @@
 flask
 flask-restful
-gitpython
 pandas
 requests
+retrying
 selenium
 simplejson
 scikit-learn
 pygments
-oauth2client==3.0
 google-api-python-client
+google
+oauth2client==3.0

setup.py

@@ -2,38 +2,96 @@
 import codecs
 import os
 
-setup(
-    # Application name:
-    name="singularity",
+##########################################################################################
+# HELPER FUNCTIONS #######################################################################
+##########################################################################################
 
-    # Version number:
-    version="0.87",
+def get_lookup():
+    '''get version by way of singularity.version, returns a 
+    lookup dictionary with several global variables without
+    needing to import singularity
+    '''
+    lookup = dict()
+    version_file = os.path.join('singularity', 'version.py')
+    with open(version_file) as filey:
+        exec(filey.read(), lookup)
+    return lookup
 
-    # Application author details:
-    author="Vanessa Sochat",
-    author_email="[email protected]",
+# Read in requirements
+def get_requirements(lookup=None):
+    '''get_requirements reads in requirements and versions from
+    the lookup obtained with get_lookup'''
 
-    # Packages
-    packages=find_packages(),
- 
-    # Data files
-    include_package_data=True,
-    zip_safe=False,
+    if lookup == None:
+        lookup = get_lookup()
 
-    # Details
-    url="http://www.github.com/singularityware/singularity-python",
+    install_requires = []
+    for module in lookup['INSTALL_REQUIRES']:
+        module_name = module[0]
+        module_meta = module[1]
+        if "exact_version" in module_meta:
+            dependency = "%s==%s" %(module_name,module_meta['exact_version'])
+        elif "min_version" in module_meta:
+            if module_meta['min_version'] == None:
+                dependency = module_name
+            else:
+                dependency = "%s>=%s" %(module_name,module_meta['min_version'])
+        install_requires.append(dependency)
+    return install_requires
 
-    license="LICENSE",
-    description="Command line tool for working with singularity-hub and packaging singularity containers.",
-    keywords='singularity containers hub reproducibility package science',
 
-    install_requires = ['Flask','gitpython','flask-restful','selenium','simplejson','scikit-learn','pygments',
-                        'requests','oauth2client','google-api-python-client','pandas'],
 
-    entry_points = {
-        'console_scripts': [
-            'shub=singularity.scripts:main',
-        ],
-    },
+# Make sure everything is relative to setup.py
+install_path = os.path.dirname(os.path.abspath(__file__)) 
+os.chdir(install_path)
 
-)
+# Get version information from the lookup
+lookup = get_lookup()
+VERSION = lookup['__version__']
+NAME = lookup['NAME']
+AUTHOR = lookup['AUTHOR']
+AUTHOR_EMAIL = lookup['AUTHOR_EMAIL']
+PACKAGE_URL = lookup['PACKAGE_URL']
+KEYWORDS = lookup['KEYWORDS']
+DESCRIPTION = lookup['DESCRIPTION']
+LICENSE = lookup['LICENSE']
+with open('README.md') as filey:
+    LONG_DESCRIPTION = filey.read()
+
+##########################################################################################
+# MAIN ###################################################################################
+##########################################################################################
+
+
+if __name__ == "__main__":
+
+    INSTALL_REQUIRES = get_requirements(lookup)
+
+    setup(name=NAME,
+          version=VERSION,
+          author=AUTHOR,
+          author_email=AUTHOR_EMAIL,
+          maintainer=AUTHOR,
+          maintainer_email=AUTHOR_EMAIL,
+          packages=find_packages(), 
+          include_package_data=True,
+          zip_safe=False,
+          url=PACKAGE_URL,
+          license=LICENSE,
+          description=DESCRIPTION,
+          long_description=LONG_DESCRIPTION,
+          keywords=KEYWORDS,
+          install_requires = INSTALL_REQUIRES,
+          classifiers=[
+              'Intended Audience :: Science/Research',
+              'Intended Audience :: Developers',
+              'License :: OSI Approved :: MIT License',
+              'Programming Language :: C',
+              'Programming Language :: Python',
+              'Topic :: Software Development',
+              'Topic :: Scientific/Engineering',
+              'Operating System :: Unix',
+              'Programming Language :: Python :: 2.7',
+              'Programming Language :: Python :: 3',
+          ],
+          entry_points = {'console_scripts': [ 'shub=singularity.scripts:main' ] })

singularity/__init__.py

@@ -0,0 +1,5 @@
+from singularity.version import __version__
+
+__all__ = ['analysis', 'build', 'views', 'api',
+           'app', 'cli', 'logman', 'scripts',
+           'utils', '__version__']

singularity/build/google.py

@@ -36,6 +36,7 @@
 import pickle
 import re
 import requests
+from retrying import retry
 import sys
 import tempfile
 import time
@@ -47,24 +48,35 @@
 from singularity.logman import bot
 
 ##########################################################################################
-# GOOGLE STORAGE API #####################################################################
+# GOOGLE GENERAL API #####################################################################
 ##########################################################################################
 
-def get_storage_service():
+def get_google_service(service_type=None,version=None):
+    '''
+    get_url will use the requests library to get a url
+    :param service_type: the service to get (default is storage)
+    :param version: version to use (default is v1)
+    '''
+    if service_type == None:
+        service_type = "storage"
+    if version == None:
+        version = "v1"
+
     credentials = GoogleCredentials.get_application_default()
-    return build('storage', 'v1', credentials=credentials)
+    return build(service_type, version, credentials=credentials) 
 
 
-def get_compute_service():
-    credentials = GoogleCredentials.get_application_default()
-    return build('compute', 'v1', credentials=credentials)
+##########################################################################################
+# GOOGLE STORAGE API #####################################################################
+##########################################################################################
 
-
+@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
 def get_bucket(storage_service,bucket_name):
     req = storage_service.buckets().get(bucket=bucket_name)
     return req.execute()
 
 
+@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
 def delete_object(storage_service,bucket_name,object_name):
     '''delete_file will delete a file from a bucket
     :param storage_service: the service obtained with get_storage_service
@@ -75,6 +87,7 @@ def delete_object(storage_service,bucket_name,object_name):
                                             object=object_name).execute()
 
 
+@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
 def upload_file(storage_service,bucket,bucket_path,file_name,verbose=True):
     '''get_folder will return the folder with folder_name, and if create=True,
     will create it if not found. If folder is found or created, the metadata is
@@ -103,6 +116,7 @@ def upload_file(storage_service,bucket,bucket_path,file_name,verbose=True):
     return request.execute()
 
 
+@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
 def list_bucket(bucket,storage_service):
     # Create a request to objects.list to retrieve a list of objects.        
     request = storage_service.objects().list(bucket=bucket['id'], 
@@ -183,7 +197,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
         params['bucket_name'] = "singularity-hub"
 
     if params['padding'] == None:
-        params['padding'] = 50
+        params['padding'] = 200
 
     output = run_build_main(build_dir=build_dir,
                             params=params)
@@ -209,7 +223,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
         bot.logger.info("Sending build files %s to storage",'\n'.join(build_files))
 
         # Start the storage service, retrieve the bucket
-        storage_service = get_storage_service()
+        storage_service = get_google_service() # default is "storage" "v1"
         bucket = get_bucket(storage_service,params["bucket_name"])
 
         # For each file, upload to storage
@@ -275,7 +289,7 @@ def finish_build(verbose=True):
     params = pickle.load(open(passing_params,'rb'))
 
     # Start the storage service, retrieve the bucket
-    storage_service = get_storage_service()
+    storage_service = get_google_service()
     bucket = get_bucket(storage_service,params['bucket_name'])
     image_path = "%s/%s" %(re.sub('^http.+//www[.]','',params['repo_url']),params['commit'])
 

singularity/build/utils.py

@@ -25,6 +25,17 @@
 import tempfile
 import zipfile
 
+######################################################################################
+# Retry Functions
+######################################################################################
+
+def stop_if_result_none(result):
+    '''stop if result none will return True if we should not retry 
+    when result is none, False otherwise using retrying python package
+    '''
+    do_retry = result is not None
+    return do_retry
+
 
 ######################################################################################
 # Build Templates
@@ -62,6 +73,11 @@ def get_build_template(template_name,params=None,to_file=None):
         return None
 
 
+######################################################################################
+# Software Versions
+######################################################################################
+
+
 def get_singularity_version(singularity_version=None):
     '''get_singularity_version will determine the singularity version for a build
     first, an environmental variable is looked at, followed by using the system
@@ -83,6 +99,12 @@ def get_singularity_version(singularity_version=None):
     return singularity_version
 
 
+
+######################################################################################
+# Extensions and Files
+######################################################################################
+
+
 def sniff_extension(file_path,verbose=True):
     '''sniff_extension will attempt to determine the file type based on the extension,
     and return the proper mimetype