Merge pull request #54 from vsoch/add/client

Add/client

Author: vsoch

examples/singularity_hub/compare_builds.py

@@ -0,0 +1,50 @@
+# Compare Singularity Hub containers
+
+# This is a simple script to use the singularity command line tool to download containers
+# (using Singularity, Section 1) and compare build specs (using Singularity Hub API, Section 2) and to
+# compare the containers themselves using singularity python (Section 3)
+
+container_names = ['vsoch/singularity-hello-world',
+                   'researchapps/quantum_state_diffusion',
+                   'vsoch/pefinder']
+
+from singularity.hub.client import Client
+from singularity.package import get_image_hash
+
+import tempfile
+import os
+import demjson
+import pandas
+import shutil
+
+shub = Client()    # Singularity Hub Client
+results = dict()
+
+# Let's keep images in a temporary folder
+storage = tempfile.mkdtemp()
+os.chdir(storage)
+
+# We will keep a table of information
+columns = ['name','build_time_seconds','hash','size','commit','estimated_os']
+df = pandas.DataFrame(columns=columns)
+
+for container_name in container_names:
+    
+    # Retrieve the container based on the name
+    collection = shub.get_collection(container_name)
+    container_ids = collection['container_set']
+    containers = []
+    for container_id in container_ids:
+       manifest = shub.get_container(container_id)
+       containers.append(manifest)
+       image = shub.pull_container(manifest,
+                                   download_folder=storage,
+                                   name="%s.img.gz" %(manifest['version']))       
+       # Get hash of file
+       hashes.append(get_image_hash(image))
+       df.loc['%s-%s' %(container_name,manifest['version'])]
+
+    results[container_name] = {'collection':collection,
+                               'containers':containers}
+
+shutil.rmtree(storage)

singularity/cli.py

@@ -126,14 +126,17 @@ def execute(self,image_path,command,writable=False,contain=False):
         if writable == True:
             sudo = True
 
-        cmd = cmd + [image_path,command]
+        if not isinstance(list,command):
+            command = command.split(' ')
+
+        cmd = cmd + [image_path] + command
 
         # Run the command
         return self.run_command(cmd,sudo=sudo)
 
 
 
-    def export(self,image_path,pipe=False,output_file=None,command=None,export_format="tar"):
+    def export(self,image_path,pipe=False,output_file=None,export_format="tar"):
         '''export will export an image, sudo must be used.
         :param image_path: full path to image
         :param pipe: export to pipe and not file (default, False)
@@ -184,8 +187,10 @@ def importcmd(self,image_path,input_source,import_type=None,command=None):
         sudo = True
         if import_type == "tar":
             cmd = ['singularity','import','--file',input_source]
-            if command != None:
-                cmd = cmd + ["--command",command]
+            if command is not None:
+                if not isinstance(list,command):
+                    command = command.split(' ')
+                cmd = cmd + ["--command"] + command
             cmd.append(image_path)
             return self.run_command(cmd,sudo=sudo)
         else:
@@ -199,7 +204,7 @@ def pull(self,image_path):
         '''pull will pull a singularity hub image
         :param image_path: full path to image
         ''' 
-        if not image_path.startswit('shub://'):
+        if not image_path.startswith('shub://'):
             bot.logger.error("pull is only valid for the shub://uri, %s is invalid.",image_name)
             sys.exit(1)           
 
@@ -211,44 +216,57 @@ def pull(self,image_path):
 
 
 
-    def run(self,image_path,command,writable=False,contain=False):
-        '''run will run a command inside the container, probably not intended for within python
+    def run(self,image_path,args=None,writable=False,contain=False):
+        '''run will run the container, with or withour arguments (which
+        should be provided in a list)
         :param image_path: full path to singularity image
-        :param command: command to send to container
+        :param args: args to include with the run
         '''
         sudo = False
         cmd = ["singularity","run"]
         cmd = self.add_flags(cmd,writable=writable,contain=contain)
+        cmd = cmd + [image_path]
 
         # Conditions for needing sudo
         if writable == True:
             sudo = True
-
-        cmd = cmd + [image_path,command]
-
-        # Run the command
-        return self.run_command(cmd,sudo=sudo)
-
-
-    def start(self,image_path,writable=False,contain=False):
-        '''start will start a container
+        
+        if args is not None:        
+            if not isinstance(list,args):
+                args = command.split(' ')
+            cmd = cmd + args
+
+        result = self.run_command(cmd,sudo=sudo)
+        if isinstance(result,bytes):
+            result = result.decode('utf-8')
+        result = result.strip('\n')
+        try:
+            result = json.loads(result)
+        except:
+            pass
+        return result
+
+
+    def get_labels(self,image_path):
+        '''get_labels will return all labels defined in the image
         '''
-        sudo = False
-        cmd = ['singularity','start']
-        cmd = self.add_flags(cmd,writable=writable,contain=contain)
-        if writable == True:
-            sudo = True
-
-        cmd.append(image_path)
-        return self.run_command(cmd,sudo=sudo)
-
-
-    def stop(self,image_path):
-        '''stop will stop a container
+        cmd = ['singularity','exec',image_path,'cat','/.singularity/labels.json']
+        labels = self.run_command(cmd)
+        return json.loads(labels.decode('utf-8'))
+        
+
+    def get_args(self,image_path):
+        '''get_args will return the subset of labels intended to be arguments
+        (in format SINGULARITY_RUNSCRIPT_ARG_*
         '''
-        cmd = ['singularity','stop',image_path]
-        return self.run_command(cmd)
-
+        labels = self.get_labels(image_path)
+        args = dict()
+        for label,values in labels.items():
+            if re.search("^SINGULARITY_RUNSCRIPT_ARG",label):
+                vartype = label.split('_')[-1].lower()
+                if vartype in ["str","float","int","bool"]:
+                    args[vartype] = values.split(',')
+        return args
 
 
     def add_flags(self,cmd,writable,contain):
@@ -267,9 +285,11 @@ def add_flags(self,cmd,writable,contain):
         return cmd
 
 
-######################################################################################################
+
+
+#################################################################################
 # HELPER FUNCTIONS
-######################################################################################################
+#################################################################################
 
 def get_image(image,return_existed=False,sudopw=None,size=None,debug=False):
     '''get_image will return the file, if it exists, or if it's docker or

singularity/hub/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+'''
+auth.py: authentication functions for singularity hub api
+         currently no token / auth for private collections
+'''
+
+from singularity.logman import bot
+import os
+import sys
+
+
+def get_headers(token=None):
+    '''get_headers will return a simple default header for a json
+    post. This function will be adopted as needed.
+    :param token: an optional token to add for auth
+    '''
+    headers = dict()
+    headers["Content-Type"] = "application/json"
+    if token!=None:
+        headers["Authorization"] = "Bearer %s" %(token)
+
+    header_names = ",".join(list(headers.keys()))
+    bot.logger.debug("Headers found: %s",header_names)
+    return headers

singularity/hub/auth.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+'''
+base.py: base module for working with singularity hub api. Right
+         now serves to hold defaults.
+
+'''
+
+from singularity.hub.utils import (
+    parse_container_name,
+    is_number,
+    api_get,
+    api_post
+)
+
+import os
+import sys
+from singularity.logman import bot
+
+api_base = "https://singularity-hub.org/api"
+
+def get_template(container_name,get_type=None):
+    '''get a container/collection or return None.
+    '''
+    if get_type == None:
+        get_type = "container"
+    get_type = get_type.lower().replace(' ','')
+
+    result = None
+    image = parse_container_name(container_name)
+    if is_number(image):
+        url = "%s/%ss/%s" %(api_base,get_type,image)
+
+    elif image['user'] is not None and image['repo_name'] is not None:        
+        url = "%s/%s/%s/%s" %(api_base,
+                              get_type,
+                              image['user'],
+                              image['repo_name'])
+
+        if image['repo_tag'] is not None and get_type is not "collection":
+            url = "%s:%s" %(url,image['repo_tag'])
+ 
+    result = api_get(url)
+
+    return result
+
+
+def download_image(manifest,download_folder=None,extract=True,name=None):
+    '''download_image will download a singularity image from singularity
+    hub to a download_folder, named based on the image version (commit id)
+    :param manifest: the manifest obtained with get_manifest
+    :param download_folder: the folder to download to, if None, will be pwd
+    :param extract: if True, will extract image to .img and return that.
+    :param name: if defined, use custom set image name instead of default
+    '''    
+    if name is not None:
+        image_file = name
+    else:
+        image_file = get_image_name(manifest)
+
+    print("Found image %s:%s" %(manifest['name'],manifest['branch']))
+    print("Downloading image... %s" %(image_file))
+
+    if download_folder != None:
+        image_file = "%s/%s" %(download_folder,image_file)
+    url = manifest['image']
+    image_file = api_get(url,stream_to=image_file)
+    if extract == True:
+        print("Decompressing %s" %image_file)
+        os.system('gzip -d -f %s' %(image_file))
+        image_file = image_file.replace('.gz','')
+    return image_file
+
+
+# Various Helpers ---------------------------------------------------------------------------------
+def get_image_name(manifest,extension='img.gz',use_hash=False):
+    '''get_image_name will return the image name for a manifest
+    :param manifest: the image manifest with 'image' as key with download link
+    :param use_hash: use the image hash instead of name
+    '''
+    if not use_hash:
+        image_name = "%s-%s.%s" %(manifest['name'].replace('/','-'),
+                                  manifest['branch'].replace('/','-'),
+                                  extension)
+    else:
+        image_url = os.path.basename(unquote(manifest['image']))
+        image_name = re.findall(".+[.]%s" %(extension),image_url)
+        if len(image_name) > 0:
+            image_name = image_name[0]
+        else:
+            bot.logger.error("Singularity Hub Image not found with expected extension %s, exiting.",extension)
+            sys.exit(1)
+            
+    bot.logger.info("Singularity Hub Image: %s", image_name)
+    return image_name

singularity/hub/base.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+'''
+client.py: simple client for singularity hub api
+
+'''
+
+from singularity.hub.auth import (
+    get_headers
+)
+
+from singularity.logman import bot
+from singularity.hub.base import (
+    get_template,
+    download_image
+)
+
+import demjson
+
+class Client(object):
+
+
+    def __init__(self, token=None):
+ 
+        if token is not None:
+            self.token = token
+        # currently not used
+        self.headers = get_headers(token=token)
+
+
+    def update_headers(self, headers):
+        '''update_headers will add headers to the client
+        :param headers: should be a dictionary of key,value to update/add to header
+        '''
+        for key,value in headers.items():
+            self.client.headers[key] = item
+
+    def load_metrics(self,manifest):
+        '''load metrics about a container build from the manifest
+        '''
+        return demjson.decode(manifest['metrics'])
+
+
+    def get_container(self,container_name):
+        '''get a container or return None.
+        '''
+        return get_template(container_name,"container")
+
+
+    def pull_container(self,manifest,download_folder=None,extract=True,name=None):
+        '''pull a container to the local machine'''
+        return download_image(manifest=manifest,
+                              download_folder=download_folder,
+                              extract=extract,
+                              name=name)
+
+
+
+    def get_collection(self,container_name):
+        '''get a container collection or return None.
+        '''
+        return get_template(container_name,"collection")