updating builder to store compressed image on cloud

Author: vsoch

setup.py

@@ -7,7 +7,7 @@
     name="singularity",
 
     # Version number:
-    version="0.30",
+    version="0.32",
 
     # Application author details:
     author="Vanessa Sochat",

singularity/build.py

@@ -24,6 +24,7 @@
 from oauth2client import client
 from oauth2client.service_account import ServiceAccountCredentials
 
+import io
 import os
 import re
 import requests
@@ -38,6 +39,8 @@
 
 # Log everything to stdout
 logging.basicConfig(stream=sys.stdout,level=logging.DEBUG)
+# if we want logging to variable or other, TBD
+#from singularity.logman import bot
 
 ##########################################################################################
 # GOOGLE STORAGE API #####################################################################
@@ -170,15 +173,19 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                                 output_folder=build_dir,
                                 build_dir=build_dir)
 
-        # Package the image
+        # Compress image
+        compressed_image = "%s.tar.gz" %image
+        os.system('sudo singularity export %s | gzip -9 > %s' %(image,compressed_image))
+        
+        # Package the image metadata (files, folders, etc)
         image_package = package(image_path=image,
                                 spec_path=spec_file,
                                 output_folder=build_dir,
                                 sudopw='',
                                 remove_image=True,
                                 verbose=True)
 
-        # Upload image to Google Storage
+        # Upload image package files to Google Storage
         if os.path.exists(image_package):
             logging.info("Package %s successfully built",image_package)
             dest_dir = "%s/build" %(build_dir)
@@ -189,14 +196,14 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
             # The path to the images on google drive will be the github url/commit folder
             image_path = "%s/%s" %(re.sub('^http.+//www[.]','',params['repo_url']),params['commit'])
             build_files = glob("%s/*" %(dest_dir))
+            build_files.append(compressed_image)
             logging.info("Sending build files %s to storage",'\n'.join(build_files))
 
             # Start the storage service, retrieve the bucket
             storage_service = get_storage_service()
             bucket = get_bucket(storage_service,bucket_name)
 
             # For each file, upload to storage
-            # TODO: here we need to skip image / layerize
             files = []
             for build_file in build_files:
                 storage_file = upload_file(storage_service,
@@ -205,7 +212,6 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                                            file_name=build_file)  
                 files.append(storage_file)
 
-
             # Upload the package as well
             package_file = upload_file(storage_service,
                                        bucket=bucket,

singularity/logman.py

@@ -0,0 +1,59 @@
+import io
+import os
+import logging
+
+class Logman:
+
+    def __init__(self,MESSAGELEVEL=None):
+        self.level = get_logging_level(MESSAGELEVEL)
+        logging.basicConfig(level=self.level)
+        self.logger = logging.getLogger('shub_builder')
+        self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        self.capture = io.StringIO()
+
+    def start(self):
+        self.ch = logging.StreamHandler(self.capture)
+        self.ch.setLevel(self.level)
+        self.ch.setFormatter(self.formatter)
+        self.logger.addHandler(self.ch)
+
+    def stop(self):
+        ### Pull the contents back into a string and close the stream
+        self.contents = self.capture.getvalue()
+        self.capture.close()
+        return self.contents
+
+
+def get_logging_level(MESSAGELEVEL=None):
+    '''get_logging_level will return a logging level based on first
+    a variable going into the function, then an environment variable
+    MESSAGELEVEL, and then the default is DEBUG.
+    :param MESSAGELEVEL: the level to get.
+    '''
+    if MESSAGELEVEL == None:
+        MESSAGELEVEL = os.environ.get("MESSAGELEVEL","DEBUG")
+
+    print("Environment message level found to be %s" %MESSAGELEVEL)
+
+    if MESSAGELEVEL == "FATAL":
+        return logging.FATAL
+
+    elif MESSAGELEVEL == "CRITICAL":
+        return logging.CRITICAL
+
+    elif MESSAGELEVEL == "ERROR":
+        return logging.ERROR
+
+    elif MESSAGELEVEL == "WARNING":
+        return logging.WARNING
+
+    elif MESSAGELEVEL == "INFO":
+        return logging.INFO
+
+    elif MESSAGELEVEL in "DEBUG":
+        return logging.DEBUG
+
+    return logging.DEBUG
+
+bot = Logman()
+bot.start()

singularity/package.py

@@ -48,11 +48,13 @@ def build_from_spec(spec=None,build_dir=None,size=None,sudopw=None,
         cli.create(image_path,size=size)
     result = cli.bootstrap(image_path=image_path,spec_path=spec_path)
     print(result)
-    # Rename the image
-    version = get_image_hash(image_path)
-    final_path = "%s/%s" %(build_dir,version)
-    os.rename(image_path,final_path)
-    return final_path
+    # If image, rename based on hash
+    if build_folder == False:
+        version = get_image_hash(image_path)
+        final_path = "%s/%s" %(build_dir,version)
+        os.rename(image_path,final_path)
+        image_path = final_path
+    return image_path
 
 
 def package(image_path,spec_path=None,output_folder=None,runscript=True,
@@ -118,7 +120,8 @@ def list_package(package_path):
     '''
     zf = zipfile.ZipFile(package_path, 'r')
     return zf.namelist()
-    
+
+
 
 def calculate_similarity(pkg1,pkg2,include_files=False,include_folders=True):
     '''calculate_similarity will calculate similarity of images in packages based on
@@ -279,7 +282,6 @@ def get_image_hash(image_path):
     if the image matches the spec file
     :param image_path: full path to the singularity image
     '''
-    print("Generating unique version of image (md5 hash)")
     hash_md5 = hashlib.md5()
     with open(image_path, "rb") as f:
         for chunk in iter(lambda: f.read(4096), b""):