adding example of plotting

Author: vsoch

examples/package_tree/calculate_similarity.py

@@ -32,7 +32,7 @@
 ###############################################################################
 
 # Option 1: Get a family manually
-package_set1 = get_packages(family='docker-os')
+package_set1 = get_packages(family='docker-library')
 
 # Option 2: Specify your own package directory (arg is packages=packages)
 package_directory = '%s/examples/package_image/packages' %(base)
@@ -46,7 +46,7 @@
 ###############################################################################
 
 # Option 1: specify another (same or different) family of packages
-package_set2 = get_packages(family='docker-library')
+package_set2 = get_packages(family='docker-os')
 
 # Option 2: Same as above
 # Option 3: Don't specify any packages, use defaults

examples/package_tree/docker-library.png

@@ -1,127 +1,10 @@
-from scipy.spatial.distance import pdist, squareform
-from plotly.tools import FigureFactory as FF
-from plotly.graph_objs import *
-import plotly.plotly as py
-from glob import glob
-import numpy as np
-import pickle
+# A quick example of making a package tree with data derived from calculate_similarity.py
 
-base = '/home/vanessa/Documents/Dropbox/Code/singularity/singularity-python'
-analysis_directory = "%s/examples/package_tree" %(base)
-
-
-# From https://plot.ly/python/dendrogram/, thanks plotly! :)
+from singularity.views.trees import make_package_tree
 
 data = pickle.load(open('comparisons.pkl','rb'))['files.txt']
-
-#lookup = {x:x.split("-")[0] for x in data.index.tolist()}
-#    labels = [lookup[x] for x in data.index.tolist()]
-
-labels = data.index.tolist()
-# Initialize figure by creating upper dendrogram
-figure = FF.create_dendrogram(data, orientation='bottom', labels=labels)
-for i in range(len(figure['data'])):
-    figure['data'][i]['yaxis'] = 'y2'
-
-# Create Side Dendrogram
-dendro_side = FF.create_dendrogram(data, orientation='right')
-for i in range(len(dendro_side['data'])):
-    dendro_side['data'][i]['xaxis'] = 'x2'
-
-# Add Side Dendrogram Data to Figure
-figure['data'].extend(dendro_side['data'])
-
-# Create Heatmap
-dendro_leaves = dendro_side['layout']['yaxis']['ticktext']
-dendro_leaves = list(map(int, dendro_leaves))
-data_dist = pdist(data)
-heat_data = squareform(data_dist)
-heat_data = heat_data[dendro_leaves,:]
-heat_data = heat_data[:,dendro_leaves]
-
-heatmap = Data([
-    Heatmap(
-        x = dendro_leaves, 
-        y = dendro_leaves,
-        z = heat_data,    
-        colorscale = 'YIGnBu'
-    )
-])
-
-heatmap[0]['x'] = figure['layout']['xaxis']['tickvals']
-heatmap[0]['y'] = dendro_side['layout']['yaxis']['tickvals']
-
-# Add Heatmap Data to Figure
-figure['data'].extend(Data(heatmap))
-
-    # Edit Layout
-    figure['layout'].update({'width':800, 'height':800,
-                             'showlegend':False, 'hovermode': 'closest',
-                             })
-
-    # Edit xaxis
-    figure['layout']['xaxis'].update({'domain': [.15, 1],
-                                      'mirror': False,
-                                      'showgrid': False,
-                                      'showline': False,
-                                      'zeroline': False,
-                                      'ticks':""})
-    # Edit xaxis2
-    figure['layout'].update({'xaxis2': {'domain': [0, .15],
-                                       'mirror': False,
-                                       'showgrid': False,
-                                       'showline': False,
-                                       'zeroline': False,
-                                       'showticklabels': False,
-                                       'ticks':""}})
-
-    # Edit yaxis
-    figure['layout']['yaxis'].update({'domain': [0, .85],
-                                      'mirror': False,
-                                      'showgrid': False,
-                                      'showline': False,
-                                      'zeroline': False,
-                                      'showticklabels': False,
-                                      'ticks': ""})
-    # Edit yaxis2
-    figure['layout'].update({'yaxis2':{'domain':[.825, .975],
-                                       'mirror': False,
-                                       'showgrid': False,
-                                       'showline': False,
-                                       'zeroline': False,
-                                       'showticklabels': False,
-                                       'ticks':""}})
-
-    # Plot!
-    py.iplot(figure, filename='dendrogram_with_heatmap')
-
-
-# AND TREE
-from matplotlib import pyplot as plt
-from scipy.cluster.hierarchy import dendrogram, linkage
-import numpy as np
-Z = linkage(data, 'ward')
-
-from scipy.cluster.hierarchy import cophenet
-from scipy.spatial.distance import pdist
-
-c, coph_dists = cophenet(Z, pdist(data))
-c
-
-
-plt.figure(figsize=(10, 8))
-plt.scatter(Z[:,0], Z[:,1])  # plot all points
+plt = make_package_tree(matrix=data)
 plt.show()
 
-labels = [x.replace(':latest','') for x in data.index.tolist()]
-plt.figure(figsize=(25, 10))
-plt.title('Docker Library Similarity')
-plt.xlabel('image index')
-plt.ylabel('distance')
-dendrogram(
-    Z,
-    leaf_rotation=90.,  # rotates the x axis labels
-    leaf_font_size=8.,  # font size for the x axis labels
-    labels=labels
-)
-plt.show()
+# or save to file
+plt.savefig('examples/package_tree/docker-library.png')

examples/package_tree/plot_similarity.py

@@ -5,6 +5,7 @@ pandas
 requests
 selenium
 simplejson
+scikit-learn
 pygments
 oauth2client
 google-api-python-client

requirements.txt

@@ -27,7 +27,7 @@
     description="Command line tool for working with singularity-hub and packaging singularity containers.",
     keywords='singularity containers hub reproducibility package science',
 
-    install_requires = ['Flask','gitpython','flask-restful','selenium','simplejson','pygments',
+    install_requires = ['Flask','gitpython','flask-restful','selenium','simplejson','scikit-learn','pygments',
                         'requests','oauth2client','google-api-python-client','pandas'],
 
     entry_points = {

setup.py

@@ -125,9 +125,9 @@ def compare_packages(packages_set1=None,packages_set2=None,by=None):
     package_folder = "%s/analysis/packages" %get_installdir()
 
     if packages_set1 == None:
-        package_set1 = glob("%s/docker-os/*.zip" %(package_folder))
+        package_set1 = glob("%s/docker-library/*.zip" %(package_folder))
     if packages_set2 == None:
-        package_set2 = glob("%s/docker-library/*.zip" %(package_folder))
+        package_set2 = glob("%s/docker-os/*.zip" %(package_folder))
 
     if by == None:
         by = ['files.txt']

singularity/analysis/compare.py

@@ -6,22 +6,36 @@
 '''
 
 import json
-import os
-import re
-import requests
+
+from matplotlib import pyplot as plt
+from scipy.cluster.hierarchy import (
+    dendrogram, 
+    linkage
+)
+
+from scipy.cluster.hierarchy import cophenet
+from scipy.spatial.distance import pdist
+import numpy as np
+
 from singularity.logman import bot
 
 from singularity.views.utils import get_container_contents
 from singularity.analysis.compare import (
+    calculate_similarity,
     compare_containers,
-    calculate_similarity
+    compare_packages
 )
 
+
 from singularity.package import (
     load_package,
     package
 )
 
+import os
+import pandas
+import re
+import requests
 import shutil
 import sys
 import tempfile
@@ -176,3 +190,41 @@ def make_container_tree(folders,files,path_delim="/",parse_files=True):
         result['files'] = file_lookup
 
     return result
+
+
+###################################################################################
+# DENDROGRAM
+###################################################################################
+
+
+def make_package_tree(matrix=None,labels=None,width=25,height=10,title=None):
+    '''make package tree will make a dendrogram comparing a matrix of packages
+    :param matrix: a pandas df of packages, with names in index and columns
+    :param labels: a list of labels corresponding to row names, will be
+    pulled from rows if not defined
+    :param title: a title for the plot, if not defined, will be left out.
+    :returns a plot that can be saved with savefig
+    '''
+    if not isinstance(matrix,pandas.DataFrame):
+        bot.log.info("No pandas DataFrame (matrix) of similarities defined, will use default.")
+        matrix = compare_packages()['files.txt']
+        title = 'Docker Library Similarity to Base OS'
+
+    Z = linkage(matrix, 'ward')
+    c, coph_dists = cophenet(Z, pdist(matrix))
+
+    if labels == None:
+        labels = matrix.index.tolist()
+
+    plt.figure(figsize=(width, height))
+
+    if title != None:
+        plt.title(title)
+
+    plt.xlabel('image index')
+    plt.ylabel('distance')
+    dendrogram(Z,
+               leaf_rotation=90.,  # rotates the x axis labels
+               leaf_font_size=8.,  # font size for the x axis labels
+               labels=labels)
+    return plt