urgent updates to fix builder

Author: vsoch

singularity/build/google.py

@@ -27,7 +27,7 @@
 
 '''
 
-from singularity.api import (
+from singularity.build.api import (
     api_get, 
     api_put
 )
@@ -300,13 +300,13 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
 
         # Did the user specify a specific log file?
         logfile = get_build_metadata(key='logfile')
-        if logfile != None:
+        if logfile is not None:
             response['logfile'] = logfile
 
-        if params['branch'] != None:
+        if params['branch'] is not None:
             response['branch'] = params['branch']
 
-        if params['token'] != None:
+        if params['token'] is not None:
             response['token'] = params['token']
 
         # Send final build data to instance

singularity/build/main.py

@@ -121,7 +121,7 @@ def run_build(build_dir,params,verbose=True):
         dockerfile = dockerfile_to_singularity(dockerfile_path='Dockerfile', 
                                                output_dir=build_dir)
 
-        if dockerfile != None:
+        if dockerfile is not None:
             bot.logger.info("""\n
                                 --------------------------------------------------------------
                                 Dockerfile
@@ -153,6 +153,7 @@ def run_build(build_dir,params,verbose=True):
                 bot.logger.info("Size estimation didn't work, using default %s",params['size'])  
 
         # START TIMING
+        os.chdir(build_dir)
         start_time = datetime.now()
         image = build_from_spec(spec_file=params['spec_file'], # default will package the image
                                 size=params['size'],
@@ -172,21 +173,26 @@ def run_build(build_dir,params,verbose=True):
         # Compress image
         compressed_image = "%s.img.gz" %image
         os.system('gzip -c -9 %s > %s' %(image,compressed_image))
+
+        # Get singularity version
+        singularity_version = get_singularity_version()
+
+        old_version = False
+        if singularity_version.startswith('2.2'):
+            old_version=True
 
         # Package the image metadata (files, folders, etc)
         image_package = package(image_path=image,
                                 spec_path=params['spec_file'],
                                 output_folder=build_dir,
                                 sudopw='',
                                 remove_image=True,
-                                verbose=True)
+                                verbose=True,
+                                old_version=old_version)
 
         # Derive software tags by subtracting similar OS
         diff = get_diff(image_package=image_package)
 
-        # Get singularity version
-        singularity_version = get_singularity_version()
-
         # Get tags for services, executables
         interesting_folders = ['init','init.d','bin','systemd']
         tags = get_tags(search_folders=interesting_folders,

singularity/cli.py

@@ -110,7 +110,7 @@ def println(self,output):
             print(output)
 
 
-    def create(self,image_path,size=None):
+    def create(self,image_path,size=None,sudo=False):
         '''create will create a a new image
         :param image_path: full path to image
         :param size: image sizein MiB, default is 1024MiB
@@ -123,7 +123,7 @@ def create(self,image_path,size=None):
             cmd = ['singularity','--debug','create','--size',str(size),image_path]
         else:
             cmd = ['singularity','create','--size',str(size),image_path]
-        output = self.run_command(cmd,sudo=False)
+        output = self.run_command(cmd,sudo=sudo)
         self.println(output)        
         if os.path.exists(image_path):
             return image_path
@@ -172,20 +172,26 @@ def execute(self,image_path,command,writable=False,contain=False):
 
 
 
-    def export(self,image_path,export_format="tar"):
+    def export(self,image_path,export_format="tar",old_version=False):
         '''export will export an image, sudo must be used.
         :param image_path: full path to image
         will generate temporary directory.
         :param export_format: the export format (only tar currently supported)
         '''
-        cmd = ['singularity','export']
+        if old_version == True:
+            sudo =True
+            tmptar = "/tmp/tmptar.tar"
+            cmd = ['singularity','export','-f','/tmp/tmptar.tar']
+        else:
+            cmd = ['singularity','export']
+            sudo = False
 
         if export_format is not "tar":
             print("Currently only supported export format is tar.")
             return None
 
         cmd.append(image_path)
-        output = self.run_command(cmd,sudo=False)
+        output = self.run_command(cmd,sudo=sudo)
         return output
 
 

singularity/package.py

@@ -36,7 +36,7 @@
 
 from singularity.cli import Singularity
 from singularity.reproduce import (
-    get_image_hash,
+    get_image_file_hash,
     get_image_hashes,
     get_memory_tar,
     extract_content
@@ -75,11 +75,11 @@ def estimate_image_size(spec_file,sudopw=None,padding=None):
     bot.logger.debug("Original image size calculated as %s",original_size)
     padded_size = original_size + padding
     bot.logger.debug("Size with padding will be %s",padded_size)
-    shutil.rmtree(image_folder)
     return padded_size
 
 
-def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,build_folder=False,debug=False):
+def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,
+                    build_folder=False,debug=False):
     '''build_from_spec will build a "spec" file in a "build_dir" and return the directory
     :param spec_file: the spec file, called "Singuarity"
     :param sudopw: the sudopw for Singularity, root should provide ''
@@ -120,14 +120,14 @@ def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,build_fo
         bot.logger.debug("build_folder is true, creating %s",image_path)
         os.mkdir(image_path)
     else:
-        cli.create(image_path,size=size)
+        cli.create(image_path,size=size,sudo=True)
 
     result = cli.bootstrap(image_path=image_path,spec_path=spec_path)
     print(result)
 
     # If image, rename based on hash
     if build_folder == False:
-        version = get_image_hash(image_path)
+        version = get_image_file_hash(image_path)
         final_path = "%s/%s" %(build_dir,version)
         os.rename(image_path,final_path)
         image_path = final_path
@@ -137,7 +137,8 @@ def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,build_fo
 
 
 def package(image_path,spec_path=None,output_folder=None,runscript=True,
-            software=True,remove_image=False,verbose=False,S=None,sudopw=None):
+            software=True,remove_image=False,verbose=False,S=None,sudopw=None,
+            old_version=False):
     '''package will take an image and generate a zip (including the image
     to a user specified output_folder.
     :param image_path: full path to singularity image file
@@ -154,7 +155,15 @@ def package(image_path,spec_path=None,output_folder=None,runscript=True,
         else:
             S = Singularity(debug=verbose) # This command will ask the user for sudo
 
-    file_obj,tar = get_memory_tar(image_path)
+    # Singularity < version 2.3
+    if old_version == False:
+        file_obj,tar = get_memory_tar(image_path)
+
+    # Singularity 2.3 and up
+    else:
+        tmptar = S.export(image_path=image_path,old_version=True)
+        tar = tarfile.open(tmptar)
+
     members = tar.getmembers()
     image_name = os.path.basename(image_path)
     zip_name = "%s.zip" %(image_name.replace(" ","_"))
@@ -171,12 +180,15 @@ def package(image_path,spec_path=None,output_folder=None,runscript=True,
         to_package['Singularity'] = singularity_spec
 
     # Package the image with an sha1, identical standard, as VERSION
-    hashes = get_image_hashes(image_path)
-    to_package["VERSION"] = hashes['REPLICATE']
-    to_package["HASHES"] = hashes
+    if old_version == False:
+        hashes = get_image_hashes(image_path)
+        to_package["HASHES"] = hashes
+        to_package["VERSION"] = hashes['IDENTICAL']
+    else:
+        to_package["VERSION"] = get_image_file_hash(image_path)
+    
 
     # Look for runscript
-
     if runscript is True:
         try:
             to_package["runscript"] = extract_content(image_path,'./singularity',cli=S)
@@ -194,7 +206,10 @@ def package(image_path,spec_path=None,output_folder=None,runscript=True,
     # Do zip up here - let's start with basic structures
     zipfile = zip_up(to_package,zip_name=zip_name,output_folder=output_folder)
     bot.logger.debug("Package created at %s" %(zipfile))
-    file_obj.close()
+
+    if old_version == False:
+        file_obj.close()
+    
     # return package to user
     return zipfile
 

singularity/version.py

@@ -22,7 +22,7 @@
 SOFTWARE.
 '''
 
-__version__ = "1.1.5"
+__version__ = "1.1.7"
 AUTHOR = 'Vanessa Sochat'
 AUTHOR_EMAIL = '[email protected]'
 NAME = 'singularity'