updating client to include additional commands for labels, etc.

Author: vsoch

examples/singularity_hub/compare_builds.py

@@ -9,16 +9,42 @@
                    'vsoch/pefinder']
 
 from singularity.hub.client import Client
+from singularity.package import get_image_hash
 
-client = Client()
+import tempfile
+import os
+import demjson
+import pandas
+import shutil
+
+shub = Client()    # Singularity Hub Client
 results = dict()
 
+# Let's keep images in a temporary folder
+storage = tempfile.mkdtemp()
+os.chdir(storage)
+
+# We will keep a table of information
+columns = ['name','build_time_seconds','hash','size','commit','estimated_os']
+df = pandas.DataFrame(columns=columns)
+
 for container_name in container_names:
 
     # Retrieve the container based on the name
-    collection = client.get_collection(container_name)
+    collection = shub.get_collection(container_name)
     container_ids = collection['container_set']
     containers = []
     for container_id in container_ids:
-       container = client.get_container(container_id)
-       containers.append(container)
+       manifest = shub.get_container(container_id)
+       containers.append(manifest)
+       image = shub.pull_container(manifest,
+                                   download_folder=storage,
+                                   name="%s.img.gz" %(manifest['version']))       
+       # Get hash of file
+       hashes.append(get_image_hash(image))
+       df.loc['%s-%s' %(container_name,manifest['version'])]
+
+    results[container_name] = {'collection':collection,
+                               'containers':containers}
+
+shutil.rmtree(storage)

singularity/hub/base.py

@@ -13,6 +13,10 @@
     api_post
 )
 
+import os
+import sys
+from singularity.logman import bot
+
 api_base = "https://singularity-hub.org/api"
 
 def get_template(container_name,get_type=None):
@@ -36,6 +40,56 @@ def get_template(container_name,get_type=None):
         if image['repo_tag'] is not None and get_type is not "collection":
             url = "%s:%s" %(url,image['repo_tag'])
 
-        result = api_get(url)
+    result = api_get(url)
 
     return result
+
+
+def download_image(manifest,download_folder=None,extract=True,name=None):
+    '''download_image will download a singularity image from singularity
+    hub to a download_folder, named based on the image version (commit id)
+    :param manifest: the manifest obtained with get_manifest
+    :param download_folder: the folder to download to, if None, will be pwd
+    :param extract: if True, will extract image to .img and return that.
+    :param name: if defined, use custom set image name instead of default
+    '''    
+    if name is not None:
+        image_file = name
+    else:
+        image_file = get_image_name(manifest)
+
+    print("Found image %s:%s" %(manifest['name'],manifest['branch']))
+    print("Downloading image... %s" %(image_file))
+
+    if download_folder != None:
+        image_file = "%s/%s" %(download_folder,image_file)
+    url = manifest['image']
+    image_file = api_get(url,stream_to=image_file)
+    if extract == True:
+        print("Decompressing %s" %image_file)
+        os.system('gzip -d -f %s' %(image_file))
+        image_file = image_file.replace('.gz','')
+    return image_file
+
+
+# Various Helpers ---------------------------------------------------------------------------------
+def get_image_name(manifest,extension='img.gz',use_hash=False):
+    '''get_image_name will return the image name for a manifest
+    :param manifest: the image manifest with 'image' as key with download link
+    :param use_hash: use the image hash instead of name
+    '''
+    if not use_hash:
+        image_name = "%s-%s.%s" %(manifest['name'].replace('/','-'),
+                                  manifest['branch'].replace('/','-'),
+                                  extension)
+    else:
+        image_url = os.path.basename(unquote(manifest['image']))
+        image_name = re.findall(".+[.]%s" %(extension),image_url)
+        if len(image_name) > 0:
+            image_name = image_name[0]
+        else:
+            bot.logger.error("Singularity Hub Image not found with expected extension %s, exiting.",extension)
+            sys.exit(1)
+            
+    bot.logger.info("Singularity Hub Image: %s", image_name)
+    return image_name

singularity/hub/client.py

@@ -10,8 +10,12 @@
 )
 
 from singularity.logman import bot
-from singularity.hub.base import get_template
+from singularity.hub.base import (
+    get_template,
+    download_image
+)
 
+import demjson
 
 class Client(object):
 
@@ -31,13 +35,27 @@ def update_headers(self, headers):
         for key,value in headers.items():
             self.client.headers[key] = item
 
+    def load_metrics(self,manifest):
+        '''load metrics about a container build from the manifest
+        '''
+        return demjson.decode(manifest['metrics'])
+
 
     def get_container(self,container_name):
         '''get a container or return None.
         '''
         return get_template(container_name,"container")
 
 
+    def pull_container(self,manifest,download_folder=None,extract=True,name=None):
+        '''pull a container to the local machine'''
+        return download_image(manifest=manifest,
+                              download_folder=download_folder,
+                              extract=extract,
+                              name=name)
+
+
+
     def get_collection(self,container_name):
         '''get a container collection or return None.
         '''

singularity/hub/utils.py

@@ -13,29 +13,44 @@
 import sys
 
 
-def api_get(url,headers=None,token=None,data=None, return_json=True):
+def api_get(url,headers=None,token=None,data=None, return_json=True, stream_to=None):
     '''api_get will use requests to get a particular url
     :param url: the url to send file to
     :param headers: a dictionary with headers for the request
     :param putdata: additional data to add to the request
     :param return_json: return json if successful
+    :param stream_to: stream the response to file
     '''
     bot.logger.debug("GET %s",url)
 
+    stream = False
+    if stream_to is not None:
+        stream = True
+
     if headers == None:
         headers = get_headers(token=token)
+
     if data == None:
         response = requests.get(url,         
-                                headers=headers)
+                                headers=headers,
+                                stream=stream)
     else:
         response = requests.get(url,         
                                 headers=headers,
-                                json=data)
+                                json=data,
+                                stream=stream)
 
-    if response.status_code == 200 and return_json:
+    if response.status_code == 200 and return_json and not stream:
         return response.json()
 
-    return response
+   
+    chunk_size = 1 << 20
+    with open(stream_to,'wb') as filey:
+        for chunk in response.iter_content(chunk_size=chunk_size):
+            filey.write(chunk)
+
+    return stream_to 
+
 
 
 def api_put(url,headers=None,token=None,data=None, return_json=True):
@@ -116,7 +131,7 @@ def parse_container_name(image):
 
     # If the user provided a number (unique id for an image), return it
     if is_number(image) == True:
-        logger.info("Numeric image ID %s found.", image)
+        bot.logger.info("Numeric image ID %s found.", image)
         return int(image)
 
     image = image.split('/')

singularity/version.py

@@ -9,6 +9,7 @@
 
 INSTALL_REQUIRES = (
 
+    ('demjson', {'min_version': None}),
     ('flask', {'min_version': '0.12'}),
     ('flask-restful', {'min_version': None}),
     ('pandas', {'min_version': '0.19.2'}),