modified: singularity/build.py

	modified:   singularity/package.py
	modified:   singularity/utils.py

Author: vsoch

singularity/build.py

@@ -7,7 +7,7 @@
 
 from singularity.api import api_get, api_put, api_post
 from singularity.boutiques import get_boutiques_json
-from singularity.package import build_from_spec
+from singularity.package import build_from_spec, package
 from singularity.utils import get_installdir, read_file, write_file, download_repo
 
 from googleapiclient.discovery import build
@@ -163,13 +163,22 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
 
     if os.path.exists(spec_file):
         logging.info("Found spec file %s in repository",spec_file)
-        image_package = build_from_spec(spec=spec_file,
-                                        name=params['commit'],
-                                        size=params['size'],
-                                        sudopw='', # with root should not need sudo
-                                        output_folder=build_dir)
 
-        # If doesn't error, run google_drive_setup and upload image
+        image = build_from_spec(spec=spec_file, # default will package the image
+                                size=params['size'],
+                                sudopw='', # with root should not need sudo
+                                output_folder=build_dir,
+                                build_dir=build_dir)
+
+        # Package the image
+        image_package = package(image_path=image,
+                                spec_path=spec_file,
+                                output_folder=build_dir,
+                                sudopw='',
+                                remove_image=True,
+                                verbose=True)
+
+        # Upload image to Google Storage
         if os.path.exists(image_package):
             logging.info("Package %s successfully built",image_package)
             dest_dir = "%s/build" %(build_dir)
@@ -187,6 +196,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
             bucket = get_bucket(storage_service,bucket_name)
 
             # For each file, upload to storage
+            # TODO: here we need to skip image / layerize
             files = []
             for build_file in build_files:
                 storage_file = upload_file(storage_service,
@@ -196,6 +206,13 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                 files.append(storage_file)
 
 
+            # Upload the package as well
+            package_file = upload_file(storage_service,
+                                       bucket=bucket,
+                                       bucket_path=image_path,
+                                       file_name=image_package)
+            files.append(package_file)
+
             # If the user has specified a log file, include with data/response
             if logfile != None:
                 log_file = upload_file(storage_service,

singularity/package.py

@@ -16,83 +16,79 @@
 import os
 
 
-def build_from_spec(spec=None,name=None,build_dir=None,size=None,sudopw=None,output_folder=None):
+def build_from_spec(spec=None,build_dir=None,size=None,sudopw=None,
+                    output_folder=None,build_folder=False):
     '''build_from_spec will build a "spec" file in a "build_dir" and return the directory
     :param spec: the spec file, called "Singuarity"
-    :parma name: the name to call the image, will go under a collection for some library/name 
     :param build_dir: the directory to build in. If not defined, will use tmpdir.
     :param size: the size of the image
     :param output_folder: where to output the image package
+    :param build_folder: "build" the image into a folder instead. Default False
     '''
     if spec == None:
         spec = "Singularity"
     if build_dir == None:
         build_dir = tempfile.mkdtemp()
     # Copy the spec to a temporary directory
     spec_path = "%s/%s" %(build_dir,spec)
-    shutil.copyfile(spec,spec_path)
+    if not os.path.exists(spec_path):
+        shutil.copyfile(spec,spec_path)
     # If name isn't provided, call it Singularity
-    if name == None:
-        name = "Singularity"
-    image_path = "%s/%s.img" %(build_dir,name)
+    image_path = "%s/image" %(build_dir)
     # Run create image and bootstrap with Singularity command line tool.
     if sudopw != None:
         cli = Singularity(sudopw=sudopw)
     else:
         cli = Singularity() # This command will ask the user for sudo
     print("\nCreating and boostrapping image...")
-    cli.create(image_path,size=size)
+    # Does the user want to "build" into a folder or image?
+    if build_folder == True:
+        os.mkdir(image_path)
+    else:
+        cli.create(image_path,size=size)
     result = cli.bootstrap(image_path=image_path,spec_path=spec_path)
     print(result)
-    # Finally, package the image.
-    print("\nPacking image...")
-    zipfile = package(image_path=image_path,
-                      name=name,
-                      output_folder=output_folder,
-                      spec_path=spec_path,
-                      verbose=True,
-                      S=cli)
-    return zipfile
+    # Rename the image
+    version = get_image_hash(image_path)
+    final_path = "%s/%s" %(build_dir,version)
+    os.rename(image_path,final_path)
+    return final_path
 
 
-def package(image_path,name=None,spec_path=None,output_folder=None,runscript=True,
-                       software=True,remove_image=False,verbose=False,S=None):
+def package(image_path,spec_path=None,output_folder=None,runscript=True,
+            software=True,remove_image=False,verbose=False,S=None,sudopw=None):
     '''package will take an image and generate a zip (including the image
     to a user specified output_folder.
     :param image_path: full path to singularity image file
-    :param name: the name for the image to be included with the collection
     :param runscript: if True, will extract runscript to include in package as runscript
     :param software: if True, will extract files.txt and folders.txt to package
     :param remove_image: if True, will not include original image in package (default,False)
     :param verbose: be verbose when using singularity --export (default,False)
     :param S: the Singularity object (optional) will be created if not required.
     '''    
+    # Run create image and bootstrap with Singularity command line tool.
     if S == None:
-        S = Singularity(verbose=verbose)
+        if sudopw != None:
+            S = Singularity(sudopw=sudopw,verbose=verbose)
+        else:
+            S = Singularity(verbose=verbose) # This command will ask the user for sudo
     tmptar = S.export(image_path=image_path,pipe=False)
     tar = tarfile.open(tmptar)
     members = tar.getmembers()
     image_name = os.path.basename(image_path)
     zip_name = "%s.zip" %(image_name.replace(" ","_"))
-
     # Include the image in the package?
     if remove_image:
         to_package = dict()
     else:
         to_package = {"files":[image_path]}
-    if name != None:
-        image_name = name
-    to_package['NAME'] = format_container_name(image_name)
-
     # If the specfile is provided, it should also be packaged
     if spec_path != None:
         singularity_spec = "".join(read_file(spec_path))
         to_package['Singularity'] = singularity_spec
-
     # Package the image with an md5 sum as VERSION
     version = get_image_hash(image_path)
     to_package["VERSION"] = version
-
     # Look for runscript
     if runscript == True:
         try:
@@ -101,21 +97,17 @@ def package(image_path,name=None,spec_path=None,output_folder=None,runscript=Tru
             runscript = runscript_file.read()
             to_package["runscript"] = runscript
             print("Found runscript!")
-
         except KeyError:
             print("No runscript found in image!")
-        
     if software == True:
         print("Adding software list to package!")
         files = [x.path for x in members if x.isfile()]
         folders = [x.path for x in members if x.isdir()]
         to_package["files.txt"] = files
         to_package["folders.txt"] = folders
-
     # Do zip up here - let's start with basic structures
     zipfile = zip_up(to_package,zip_name=zip_name,output_folder=output_folder)
     print("Package created at %s" %(zipfile))
-
     # return package to user
     return zipfile
 

singularity/utils.py

@@ -108,6 +108,26 @@ def run_command(cmd,error_message=None,sudopw=None,suppress=False):
 ## FILE OPERATIONS #########################################################
 ############################################################################
 
+def zip_dir(zip_dir, zip_name, output_folder=None):
+    '''zip_dir will zip up and entire zip directory
+    :param folder_path: the folder to zip up
+    :param zip_name: the name of the zip to return
+    :output_folder:
+    '''
+    tmpdir = tempfile.mkdtemp()
+    output_zip = "%s/%s" %(tmpdir,zip_name)
+    zf = zipfile.ZipFile(output_zip, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
+    for root, dirs, files in os.walk(zip_dir):
+        for file in files:
+            zf.write(os.path.join(root, file))
+    zf.close()
+    if output_folder != None:
+        shutil.copyfile(output_zip,"%s/%s"%(output_folder,zip_name))
+        shutil.rmtree(tmpdir)
+        output_zip = "%s/%s"%(output_folder,zip_name)
+    return output_zip
+
+
 def zip_up(file_list,zip_name,output_folder=None):
     '''zip_up will zip up some list of files into a package (.zip)
     :param file_list: a list of files to include in the zip.