Merge pull request #51 from vsoch/master

adding interactive tree for singularity hub

Author: vsoch

singularity/analysis/classify.py

@@ -22,6 +22,8 @@
 from singularity.package import package as make_package
 from singularity.utils import (
     get_installdir,
+    remove_uri,
+    read_file,
     update_dict,
     update_dict_sum
 )
@@ -175,6 +177,33 @@ def get_tags(container=None,image_package=None,sudopw=None,search_folders=None,d
 ###################################################################################
 
 
+def get_files(container,S=None,tmpdir=None):
+    '''get_files will return a list of files inside a container, sorted by name
+    :param container: the container to use, either shub:// or docker:// or actual
+    '''
+    files = None
+    if tmpdir == None:
+        tmpdir = tempfile.mkdtemp()
+    tmpfile = "%s/files.txt" %tmpdir
+    container_name = remove_uri(container)
+    command = ' ls -LR >> %s 2>/dev/null' %(tmpfile)
+    if S==None:
+        S = Singularity(sudo=None)
+    result = S.execute(container,command)
+    if os.path.exists(tmpfile):
+        os.system("sed -i '/^$/d' %s" %(tmpfile))
+        os.system('sort %s -or %s' %(tmpfile,tmpfile))
+        files = read_file(tmpfile)
+        if len(files) > 0:
+            files = [x for x in files if x.startswith('.')]
+            files = [x.split(container_name)[1:] for x in files]
+            files = [x for x in files if len(x) > 0]
+            files = [x[0] for x in files]    
+    shutil.rmtree(tmpdir)
+    return files
+
+        
+
 def file_counts(container=None,patterns=None,image_package=None,sudopw=None,diff=None):
     '''file counts will return a list of files that match one or more regular expressions.
     if no patterns is defined, a default of readme is used. All patterns and files are made

singularity/analysis/compare.py

@@ -75,6 +75,7 @@ def container_similarity_vector(container1=None,packages_set=None,by=None,custom
     return comparisons
 
 
+
 def compare_containers(container1=None,container2=None,by=None,
                        image_package1=None,image_package2=None):
     '''compare_containers will generate a data structure with common and unique files to
@@ -105,19 +106,29 @@ def compare_containers(container1=None,container2=None,by=None,
     # Do the comparison for each metric
     comparisons = dict()
     for b in by:
-        intersect = list(set(container1_guts[b]).intersection(container2_guts[b]))
-        unique1 = list(set(container1_guts[b]).difference(container2_guts[b]))
-        unique2 = list(set(container2_guts[b]).difference(container1_guts[b]))
-
-        # Return data structure
-        comparison = {"intersect":intersect,
-                      "unique1": unique1,
-                      "unique2": unique2,
-                      "total1": len(container1_guts[b]),
-                      "total2": len(container2_guts[b])}
-        comparisons[b] = comparison 
+        comparisons[b] = compare_lists(container1_guts[b],container2_guts[b])
 
     return comparisons
+
+
+def compare_lists(list1,list2):
+    '''compare lists is the lowest level that drives compare_containers and
+    compare_packages. It returns a comparison object (dict) with the unique,
+    total, and intersecting things between two lists
+    :param list1: the list for container1
+    :param list2: the list for container2
+    '''
+    intersect = list(set(list1).intersection(list2))
+    unique1 = list(set(list1).difference(list2))
+    unique2 = list(set(list2).difference(list1))
+
+    # Return data structure
+    comparison = {"intersect":intersect,
+                  "unique1": unique1,
+                  "unique2": unique2,
+                  "total1": len(list1),
+                  "total2": len(list2)}
+    return comparison
 
 
 def calculate_similarity(container1=None,container2=None,image_package1=None,

singularity/build/converter.py

@@ -208,15 +208,17 @@ def get_mapping():
     #  Docker : Singularity
     add_command = {"section": "%post","fun": parse_add, "json": True }
     copy_command = {"section": "%post", "fun": parse_add, "json": True }  
-    cmd_command = {"section": "%runscript", "fun": parse_cmd, "json": True }  
+    cmd_command = {"section": "%post", "fun": parse_comment, "json": True }  
+    label_command = {"section": "%post", "fun": parse_comment, "json": True }  
+    port_command = {"section": "%post", "fun": parse_comment, "json": True }  
     env_command = {"section": "%post", "fun": parse_env, "json": False }
     comment_command = {"section": "%post", "fun": parse_comment, "json": False }
     from_command = {"section": "From", "json": False }
     run_command = {"section": "%post", "json": True}       
     workdir_command = {"section": "%post","fun": parse_workdir, "json": False }  
-    entry_command = {"section": "%post", "fun": parse_entry, "json": True }
+    entry_command = {"section": "%runscript", "fun": parse_entry, "json": True }
 
-    return {"ADD": add_command,
+    return {"ADD":add_command,
             "COPY":copy_command,
             "CMD":cmd_command,
             "ENTRYPOINT":entry_command,
@@ -225,7 +227,9 @@ def get_mapping():
             "RUN":run_command,
             "WORKDIR":workdir_command,
             "MAINTAINER":comment_command,
-            "VOLUME":comment_command}
+            "VOLUME":comment_command,
+            "EXPOSE":port_command,
+            "LABEL":label_command}
 
 
 
@@ -352,7 +356,7 @@ def print_sections(sections,mapping=None):
         mapping = get_mapping()
 
     finished_spec = None
-    ordering = ['bootstrap',"From","%runscript","%post"]
+    ordering = ['bootstrap',"From","%runscript","%post",'%test']
 
     for section in ordering:
 

singularity/cli.py

@@ -27,7 +27,8 @@
 
 class Singularity:
 
-    def __init__(self,sudo=True,sudopw=None,debug=False):
+
+    def __init__(self,sudo=False,sudopw=None,debug=False):
        '''upon init, store user password to not ask for it again'''
 
        self.sudopw = sudopw
@@ -117,7 +118,7 @@ def execute(self,image_path,command,writable=False,contain=False):
         if self.debug == True:
             cmd = ["singularity",'--debug',"exec"]
         else:
-            cmd = ["singularity","exec"]
+            cmd = ["singularity",'--quiet',"exec"]
 
         cmd = self.add_flags(cmd,writable=writable,contain=contain)
 
@@ -194,6 +195,21 @@ def importcmd(self,image_path,input_source,import_type=None,command=None):
         return None
 
 
+    def pull(self,image_path):
+        '''pull will pull a singularity hub image
+        :param image_path: full path to image
+        ''' 
+        if not image_path.startswit('shub://'):
+            bot.logger.error("pull is only valid for the shub://uri, %s is invalid.",image_name)
+            sys.exit(1)           
+
+        if self.debug == True:
+            cmd = ['singularity','--debug','pull',image_path]
+        else:
+            cmd = ['singularity','pull',image_path]
+        return self.run_command(cmd)
+
+
 
     def run(self,image_path,command,writable=False,contain=False):
         '''run will run a command inside the container, probably not intended for within python
@@ -213,6 +229,7 @@ def run(self,image_path,command,writable=False,contain=False):
         # Run the command
         return self.run_command(cmd,sudo=sudo)
 
+
     def start(self,image_path,writable=False,contain=False):
         '''start will start a container
         '''
@@ -276,7 +293,7 @@ def get_image(image,return_existed=False,sudopw=None,size=None,debug=False):
             cli = Singularity(debug=debug) # This command will ask the user for sudo
 
         tmpdir = tempfile.mkdtemp()
-        image_name = "%s.img" %image.replace("docker://","") 
+        image_name = "%s.img" %image.replace("docker://","").replace("/","-")
         bot.logger.info("Found docker image %s, creating and importing...",image_name)
         image_path = "%s/%s" %(tmpdir,image_name)
         cli.create(image_path,size=size)

singularity/scripts.py

@@ -40,6 +40,11 @@ def get_parser():
                         help="package a singularity container for singularity hub", 
                         default=False, action='store_true')
 
+    # Does the user want to package an image?
+    parser.add_argument('--remove-image', dest="remove_image", 
+                        help="remove image file from the package", 
+                        default=False, action='store_true')
+
     # Does the user want to estimate the os?
     parser.add_argument('--os', dest="os", 
                         help="estimate the operating system of your container.", 
@@ -165,7 +170,8 @@ def main():
                package(image_path=image,
                        output_folder=output_folder,
                        runscript=True,
-                       software=True)
+                       software=True,
+                       remove_image=args.remove_image)
            else:
                print("Not sure what to do?")
                parser.print_help()

singularity/utils.py

@@ -11,6 +11,7 @@
 import requests
 
 import shutil
+import json
 import simplejson
 import singularity.__init__ as hello
 from singularity.logman import bot
@@ -225,6 +226,15 @@ def read_file(filename,mode="r"):
     return content
 
 
+def read_json(filename,mode='r'):
+    '''read_json reads in a json file and returns
+    the data structure as dict.
+    '''
+    with open(filename,mode) as filey:
+        data = json.load(filey)
+    return data
+
+
 ############################################################################
 ## OTHER MISC. #############################################################
 ############################################################################
@@ -304,6 +314,12 @@ def format_container_name(name,special_characters=None):
     return ''.join(e.lower() for e in name if e.isalnum() or e in special_characters)
 
 
+def remove_uri(container):
+    '''remove_uri will remove docker:// or shub:// from the uri
+    '''
+    return container.replace('docker://','').replace('shub://','')
+
+
 def download_repo(repo_url,destination,commit=None):
     '''download_repo
     :param repo_url: the url of the repo to clone from

singularity/version.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.8"
+__version__ = "1.1.1"
 AUTHOR = 'Vanessa Sochat'
 AUTHOR_EMAIL = '[email protected]'
 NAME = 'singularity'

singularity/views/trees.py

@@ -4,7 +4,7 @@
 singularity views/trees.py: part of singularity package
 
 '''
-
+from functools import reduce
 import json
 
 from singularity.logman import bot
@@ -227,3 +227,63 @@ def make_package_tree(matrix=None,labels=None,width=25,height=10,title=None):
                leaf_font_size=8.,  # font size for the x axis labels
                labels=labels)
     return plt
+
+
+def make_interactive_tree(matrix=None,labels=None):
+    '''make interactive tree will return complete html for an interactive tree
+    :param title: a title for the plot, if not defined, will be left out.
+    '''
+    from scipy.cluster.hierarchy import (
+        dendrogram, 
+        linkage,
+        to_tree
+    )
+
+    d3 = None
+    from scipy.cluster.hierarchy import cophenet
+    from scipy.spatial.distance import pdist
+
+    if isinstance(matrix,pandas.DataFrame):
+        Z = linkage(matrix, 'ward') # clusters
+        T = to_tree(Z, rd=False)
+
+        if labels == None:
+            labels = matrix.index.tolist()
+        lookup = dict(zip(range(len(labels)), labels))
+
+        # Create a dendrogram object without plotting
+        dend = dendrogram(Z,no_plot=True,
+                      orientation="right",
+                      leaf_rotation=90.,  # rotates the x axis labels
+                      leaf_font_size=8.,  # font size for the x axis labels
+                      labels=labels)
+
+        d3 = dict(children=[], name="root")
+        add_node(T, d3)
+        label_tree(d3["children"][0],lookup)
+    else:
+        bot.logger.warning('Please provide data as pandas Data Frame.')
+    return d3
+
+
+def add_node(node, parent):
+    '''add_node will add a node to it's parent
+    '''
+    newNode = dict(node_id=node.id, children=[])
+    parent["children"].append(newNode)
+    if node.left: add_node(node.left, newNode)
+    if node.right: add_node(node.right, newNode)
+
+
+def label_tree(n,lookup):
+    '''label tree will again recursively label the tree
+    :param n: the root node, usually d3['children'][0]
+    :param lookup: the node/id lookup
+    '''
+    if len(n["children"]) == 0:
+        leaves = [lookup[n["node_id"]]]
+    else:
+        leaves = reduce(lambda ls, c: ls + label_tree(c,lookup), n["children"], [])
+    del n["node_id"]
+    n["name"] = name = "|||".join(sorted(map(str, leaves)))
+    return leaves