updating all examples (minus original paper)

Signed-off-by: Vanessa Sochat <[email protected]>

Author: vsoch

examples/classify/classify_image/count_files.py

@@ -7,13 +7,11 @@
     extension_counts
 )
 
-container = "ubuntu.simg"
+# singularity pull docker://busybox
+container = "busybox_latest.sif"
 
 # Now we might be interested in counting different things
-readme_count = file_counts(container)
-copyright_count = file_counts(container, patterns=['copyright'])
-authors_count = file_counts(container, patterns=['authors','thanks','credit'])
-todo_count = file_counts(container, patterns=['todo'])
+bin_count = file_counts(container, patterns=['bin'])
 
 # Or getting a complete dict of extensions
 extensions = extension_counts(container)

examples/classify/classify_image/derive_tags.py

@@ -15,9 +15,9 @@
 # Default tags will be returned as software in "bin"
 tags = get_tags(container)
 
-# We can also get the raw list of flies
-file_list = get_container_contents(container)['all']
-
 # We can specify other folders of interest
 folders = ['init','init.d','bin','systemd']
 tags = get_tags(container, search_folders=folders)
+
+# We can also get the raw list of flies
+file_list = get_container_contents(container)['all']

examples/reproducibility/assess_replication.py

@@ -1,34 +1,23 @@
 from glob import glob
 
-from singularity.analysis.reproduce import assess_differences
+from singularity.analysis.reproduce import assess_differences, get_level
 
-image_files=glob('*.img')
+# singularity pull docker://ubuntu:14.04
+# singularity pull docker://ubuntu:12.04
 
-# ASSESS DIFFERENCES #######################################
-# returns dictionary with 
-
-report = assess_differences(image_files[0],image_files[1])
+image_files = glob('ubuntu*.sif')
 
-report.keys()
-# dict_keys(['different', 'missing', 'same'])
 
-# These files are equivalent between the images
-print(len(report['same']))
-5663
+# Choose a level that you want to assess based on
+level_filter = {"RECIPE": get_level('RECIPE')}
 
-# These files are present in both, but different
-print(report['different'])
-['./etc/hosts', 
- './.exec', 
- './environment', 
- './etc/mtab', 
- './etc/resolv.conf', 
- './.run', 
- './.shell', 
- './singularity']
+# ASSESS DIFFERENCES #######################################
 
-# These files are found in the first image, but not the second
-print(report['missing'])
-['./var/lib/apt/lists/.wh.archive.ubuntu.com_ubuntu_dists_xenial-updates_main_i18n_Translation-en', 
- './bin/gunzip']
+# Running for all levels, this will take a few minutes
+report = assess_differences(image_files[0], image_files[1], levels=level_filter)
 
+# {'RECIPE': {'difference': [],
+#  'intersect_different': [],
+#  'same': 7,
+#  'union': 14},
+# 'scores': {'RECIPE': 1.0}}

examples/reproducibility/generate_image_hash.py

@@ -7,7 +7,7 @@
     get_image_file_hash    
 )
 
-image_files = glob("*.simg")
+image_files = glob("*.sif")
 image_path = image_files[0]
 
 ########################################################
@@ -30,7 +30,7 @@
 # We can, then generate an image hash, and by default the level "REPLICATION" will be used:
 
 get_image_hash(image_path)
-#'bf8e242931e25ae9496015868ab2e8cc8d156ffd'
+# '4c252c8fb818e4b854a478a1a0df5991'
 
 # But we can also specify a level that we want:
 get_image_hash(image_path,level="IDENTICAL")
@@ -45,10 +45,10 @@
 # of one container at one level!
 digest = get_content_hashes(image_path)
 digest['hashes']['/usr/bin/chfn']
-# 'ee2b438c278011bdac1a3a927e2d37519a8ed9c7'
+# '4b5ee4db88c3b8bfb0cb7cb3a90a7793'
 
 # We can also get a hash of the entire image file, this is done on the
 # binary file and not contents inside.
 
 file_hash = get_image_file_hash(image_path)
-# e'13775a83962ae60744d691eb7f7fd1e96599e656'
+# 'd5349c37fdc2e6f2dca8793732e1c420'

examples/shub/paper/README.md

@@ -1,4 +1,13 @@
-
 # Singularity Hub Paper
 
 This is the code for the Singularity Hub paper. You should be able to view the comparison tree at [result/index.html](result/index.html).
+Note that the version of Singularity Python used for this is older than currently on master,
+you will need to install:
+
+```bash
+git clone -b v2.5 https://www.github.com/singularityhub/singularity-python.git
+cd singularity-python
+python setup.py install
+```
+
+And the images used were also Singularity version 2.2 and 2.3.

singularity/analysis/reproduce/__init__.py

@@ -16,7 +16,6 @@
 from .utils import (
     extract_content,
     delete_image_tar,
-    get_memory_tar,
     get_image_tar
 )
 

singularity/analysis/reproduce/hash.py

@@ -164,7 +164,7 @@ def get_content_hashes(image_path,
         file_filter = level_filter
 
     elif level is None:
-        file_filter = get_level("REPLICATE",version=version,
+        file_filter = get_level("REPLICATE", version=version,
                                 skip_files=skip_files,
                                 include_files=include_files)
 

singularity/analysis/reproduce/levels.py

@@ -61,7 +61,7 @@ def get_custom_level(regexp=None,description=None,skip_files=None,include_files=
     return custom
 
 
-def get_level(level,version=None,include_files=None,skip_files=None):
+def get_level(level, version=None, include_files=None, skip_files=None):
     '''get_level returns a single level, with option to customize files
     added and skipped.
     '''

singularity/analysis/reproduce/metrics.py

@@ -1,8 +1,6 @@
 '''
 
-Copyright (C) 2017 The Board of Trustees of the Leland Stanford Junior
-University.
-Copyright (C) 2016-2017 Vanessa Sochat.
+Copyright (C) 2016-2019 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by
@@ -19,6 +17,7 @@
 
 '''
 
+from spython.utils import get_singularity_version
 from spython.main import Client
 from singularity.logger import bot
 from .levels import get_levels
@@ -47,6 +46,11 @@ def assess_differences(image_file1,
     reports = dict()
     scores = dict()
 
+    # For version 3, export sandboxes
+    if 'version 3' in get_singularity_version():
+        image_file1 = Client.export(image_file1)
+        image_file2 = Client.export(image_file2)
+
     for level_name, level_filter in levels.items():
         contenders = []
         different = []
@@ -64,22 +68,25 @@ def assess_differences(image_file1,
             guts2 = get_content_hashes(image_path=image_file2,
                                        level_filter=level_filter)
 
-        print(level_name)
         files = list(set(list(guts1['hashes'].keys()) + list(guts2['hashes'].keys())))
 
         for file_name in files:
 
-            # If it's not in one or the other
+            # If it's not in one or the other, we can't directly compare
             if file_name not in guts1['hashes'] or file_name not in guts2['hashes']:
                 setdiff.append(file_name)
 
             else:
+
+                # We can directly compare - and they are the same
                 if guts1['hashes'][file_name] == guts2['hashes'][file_name]:
                     same+=1
+
                 else:
 
                     # If the file is root owned, we compare based on size
                     if size_heuristic == True:
+
                         if guts1['root_owned'][file_name] or guts2['root_owned'][file_name]:
                             if guts1['sizes'][file_name] == guts2['sizes'][file_name]:    
                                 same+=1
@@ -88,28 +95,39 @@ def assess_differences(image_file1,
                         else:
                             # Otherwise, we can assess the bytes content by reading it
                             contenders.append(file_name)
+
+                    # We don't use a size hueristic, we just will compare based on bytes
                     else:
                         contenders.append(file_name)
 
         # If the user wants identical (meaning extraction order and timestamps)
         if level_name == "IDENTICAL":
-                different = different + contenders
+            different = different + contenders
 
         # Otherwise we need to check based on byte content
         else:        
             if len(contenders) > 0:
+
                 for rogue in contenders:
-                    hashy1 = extract_content(image_file1, rogue, return_hash=True)
-                    hashy2 = extract_content(image_file2, rogue, return_hash=True)
-        
+
+                    if 'version 3' in get_singularity_version():
+                        hashy1 = extract_content(image_file1 + rogue, return_hash=True)
+                        hashy2 = extract_content(image_file2 + rogue, return_hash=True)
+                    else:
+                        hashy1 = extract_content(rogue, return_hash=True)
+                        hashy2 = extract_content(rogue, return_hash=True)        
+
                     # If we can't compare, we use size as a heuristic
                     if hashy1 is None or hashy2 is None: # if one is symlink, could be None
-                        different.append(file_name)                    
+                        different.append(file_name)
+                    
+                    # We still fall back to size heuristic if not possible
                     elif len(hashy1) == 0 or len(hashy2) == 0:
                         if guts1['sizes'][file_name] == guts2['sizes'][file_name]:    
                             same+=1
                         else:
                             different.append(file_name)                    
+
                     elif hashy1 != hashy2:
                         different.append(rogue)
                     else:

singularity/analysis/reproduce/utils.py

@@ -1,8 +1,6 @@
 '''
 
-Copyright (C) 2017 The Board of Trustees of the Leland Stanford Junior
-University.
-Copyright (C) 2016-2017 Vanessa Sochat.
+Copyright (C) 2016-2019 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by
@@ -43,8 +41,14 @@ def extract_guts(image_path,
                  tag_root=True,
                  include_sizes=True):
 
-    '''extract the file guts from an in memory tarfile. The file is not closed.
-       This should not be done for large images.
+    '''extract the file guts from an image. 
+
+       Parameters
+       ==========
+       image_path: can be a tar, a Singularity image (sif) or a sandbox
+       file_filter: the file filter to extract guts for.
+       tag_root: if True (default) include if root owned or not.
+       include_sizes: include content sizes (defaults to True)
     '''
     if file_filter is None:
         file_filter = get_level('IDENTICAL')
@@ -59,33 +63,44 @@ def extract_guts(image_path,
     if include_sizes: 
         sizes = dict()
 
-    # Export the image
-    sandbox = Client.export(image_path)
+    # Option 1: We are given a sandbox
+    if os.path.isdir(image_path):
+        sandbox = image_path
+
+    # Option 2: it's not a sandbox, and we need to export.
+    elif 'version 3' in get_singularity_version():
+        sandbox = Client.export(image_path)
+    else:
+        sandbox = Client.image.export(image_path)
 
     # If it's tar, extract
-    if sandbox.endswith('tar'):
+    if os.path.isfile(sandbox) and sandbox.endswith('tar'):
         with tarfile.open(sandbox) as tar:
             sandbox = os.path.join(os.path.dirname(sandbox), 'sandbox') 
             tar.extractall(path=sandbox)
 
     # Recursively walk through sandbox
     for root, dirnames, filenames in os.walk(sandbox):
         for filename in filenames:
-            member_name = os.path.join(root, filename)
+            sandbox_name = os.path.join(root, filename)
+
+            # Remove the sandbox base
+            member_name = sandbox_name.lstrip(sandbox)
+
             allfiles.append(member_name)
             included = False
 
             # Skip over directories and symbolic links
-            if os.path.isdir(member_name) or os.path.islink(member_name):
+            if os.path.isdir(sandbox_name) or os.path.islink(sandbox_name):
                 continue
 
             # If we have flagged to include, and not flagged to skip
-            elif assess_content(member_name, file_filter):
-                digest[member_name] = extract_content(member_name, return_hash=True)
+            elif assess_content(sandbox_name, file_filter):
+                digest[member_name] = extract_content(sandbox_name, return_hash=True)
                 included = True
-            elif include_file(member_name, file_filter):
+            elif include_file(sandbox_name, file_filter):
                 hasher = hashlib.md5()
-                with open(member_name, 'rb') as filey:
+                with open(sandbox_name, 'rb') as filey:
                     buf = filey.read()
                     hasher.update(buf)
                 digest[member_name] = hasher.hexdigest()
@@ -94,9 +109,9 @@ def extract_guts(image_path,
             # Derive size, and if root owned
             if included:
                 if include_sizes:
-                    sizes[member_name] = os.stat(member_name).st_size
+                    sizes[member_name] = os.stat(sandbox_name).st_size
                 if tag_root:
-                    roots[member_name] = is_root_owned(member_name)
+                    roots[member_name] = is_root_owned(sandbox_name)
 
     results['all'] = allfiles
     results['hashes'] = digest
@@ -107,16 +122,6 @@ def extract_guts(image_path,
     return results
 
 
-
-def get_memory_tar(image_path):
-    '''get an in memory tar of an image. Use carefully, not as reliable
-       as get_image_tar
-    '''
-    byte_array = Client.export(image_path)
-    file_object = io.BytesIO(byte_array)
-    tar = tarfile.open(mode="r|*", fileobj=file_object)
-    return (file_object, tar)
-
 def create_tarfile(source_dir, output_filename=None):
     ''' create a tarfile from a source directory'''
     if output_filename == None: