fixing bugs for builder

Author: vsoch

singularity/analysis/reproduce/hash.py

@@ -26,7 +26,7 @@
 
 '''
 
-from singularity.cli import Singularity
+from spython.main import Client
 from singularity.logger import bot
 from singularity.analysis.reproduce.criteria import *
 from singularity.analysis.reproduce.levels import *
@@ -46,7 +46,7 @@
 
 
 
-def get_image_hashes(image_path,version=None,levels=None):
+def get_image_hashes(image_path, version=None, levels=None):
     '''get_image_hashes returns the hash for an image across all levels. This is the quickest,
     easiest way to define a container's reproducibility on each level.
     '''
@@ -98,7 +98,6 @@ def get_image_hash(image_path,
                                 skip_files=skip_files,
                                 include_files=include_files)
 
-    cli = Singularity()
     file_obj, tar = get_image_tar(image_path)
     hasher = hashlib.md5()
 
@@ -109,7 +108,7 @@ def get_image_hash(image_path,
         if member.isdir() or member.issym():
             continue
         elif assess_content(member,file_filter):
-            content = extract_content(image_path,member.name,cli)
+            content = extract_content(image_path,member.name)
             hasher.update(content)
         elif include_file(member,file_filter):
             buf = member.tobuf()

singularity/analysis/reproduce/metrics.py

@@ -26,7 +26,7 @@
 
 '''
 
-from singularity.cli import Singularity
+from spython.main import Client
 from singularity.logger import bot
 from .levels import get_levels
 from .utils import extract_content
@@ -105,10 +105,9 @@ def assess_differences(image_file1,
         # Otherwise we need to check based on byte content
         else:        
             if len(contenders) > 0:
-                cli = Singularity()
                 for rogue in contenders:
-                    hashy1 = extract_content(image_file1,rogue,cli,return_hash=True)
-                    hashy2 = extract_content(image_file2,rogue,cli,return_hash=True)
+                    hashy1 = extract_content(image_file1, rogue, return_hash=True)
+                    hashy2 = extract_content(image_file2, rogue, return_hash=True)
 
                     # If we can't compare, we use size as a heuristic
                     if hashy1 is None or hashy2 is None: # if one is symlink, could be None
@@ -140,4 +139,3 @@ def assess_differences(image_file1,
 
     reports['scores'] = scores
     return reports
-

singularity/analysis/reproduce/utils.py

@@ -26,7 +26,7 @@
 
 '''
 
-from singularity.cli import Singularity
+from spython.main import Client
 from .criteria import (
     assess_content, 
     include_file,
@@ -48,7 +48,6 @@ def extract_guts(image_path,tar,file_filter,tag_root=True,include_sizes=True):
     read the memory in tar, or from fileystem
     '''
 
-    cli = Singularity()
     results = dict()
     digest = dict()
 
@@ -64,7 +63,7 @@ def extract_guts(image_path,tar,file_filter,tag_root=True,include_sizes=True):
         if member.isdir() or member.issym():
             continue
         elif assess_content(member,file_filter):
-            digest[member_name] = extract_content(image_path,member.name,cli,return_hash=True)
+            digest[member_name] = extract_content(image_path, member.name, return_hash=True)
             included = True
         elif include_file(member,file_filter):
             hasher = hashlib.md5()
@@ -91,21 +90,18 @@ def get_memory_tar(image_path):
     '''get an in memory tar of an image. Use carefully, not as reliable
        as get_image_tar
     '''
-    cli = Singularity()
-    byte_array = cli.export(image_path)
+    byte_array = Client.export(image_path)
     file_object = io.BytesIO(byte_array)
     tar = tarfile.open(mode="r|*", fileobj=file_object)
     return (file_object,tar)
 
 
-def get_image_tar(image_path,S=None):
+def get_image_tar(image_path):
     '''get an image tar, either written in memory or to
     the file system. file_obj will either be the file object,
     or the file itself.
     '''
     bot.debug('Generate file system tar...')   
-    if S is None:
-        S = Singularity()
     file_obj = S.export(image_path=image_path)
     if file_obj is None:
         bot.error("Error generating tar, exiting.")
@@ -128,17 +124,15 @@ def delete_image_tar(file_obj, tar):
     return deleted
 
 
-def extract_content(image_path,member_name,cli=None,return_hash=False):
+def extract_content(image_path, member_name, return_hash=False):
     '''extract_content will extract content from an image using cat.
     If hash=True, a hash sum is returned instead
     '''
     if member_name.startswith('./'):
         member_name = member_name.replace('.','',1)
     if return_hash:
         hashy = hashlib.md5()
-    if cli == None:
-        cli = Singularity()
-    content = cli.execute(image_path,'cat %s' %(member_name))
+    content = Client.execute(image_path,'cat %s' %(member_name))
     if not isinstance(content,bytes):
         content = content.encode('utf-8')
         content = bytes(content)

singularity/build/scripts/singularity-prepare-instance.sh

@@ -14,6 +14,7 @@ sudo apt-get -y install git \
                    libtool \
                    squashfs-tools \
                    autotools-dev \
+                   libarchive-dev \
                    automake \
                    autoconf \
                    debootstrap \
@@ -26,6 +27,7 @@ sudo apt-get -y install git \
 
 # Pip3 installs
 sudo pip3 install --upgrade pip &&
+sudo pip3 install pyasn1-modules -U &&
 sudo pip3 install --upgrade google-api-python-client &&
 sudo pip3 install --upgrade google &&
 sudo pip3 install oauth2client==3.0.0

singularity/package/clone.py

@@ -25,15 +25,15 @@
 
 '''
 
+from spython.main import Client
 import tempfile
 from singularity.logger import bot
-from singularity.cli import Singularity
 from singularity.utils import run_command
 import platform
 import os
 import sys
 
-def package_node(root=None,name=None):
+def package_node(root=None, name=None):
     '''package node aims to package a (present working node) for a user into
     a container. This assumes that the node is a single partition.
   
@@ -81,10 +81,9 @@ def unpack_node(image_path,name=None,output_folder=None,size=None):
 
     bot.debug("Preparing to unpack %s to %s." %(image_name,name))
     unpacked_image = "%s/%s" %(output_folder,name)
-
-    S = Singularity(sudo=True) 
-    S.create(image_path=unpacked_image,
-             size=size)
+ 
+    if not os.path.exists(unpacked_image):
+        os.mkdir(unpacked_image)
 
     cmd = ["gunzip","-dc",image_path,"|","sudo","singularity","import", unpacked_image]
     output = run_command(cmd)

singularity/version.py

@@ -32,7 +32,7 @@
 LICENSE = "LICENSE"
 
 INSTALL_REQUIRES = (
-    ('spython', {'min_version': '0.0.31'}),
+    ('spython', {'min_version': '0.0.32'}),
     ('requests', {'min_version': '2.18.4'}),
     ('retrying', {'min_version': '1.3.3'}),
     ('pygments', {'min_version': '2.1.3'}),