Merge pull request #39 from vsoch/master

modify build parameters and calculate size dynamically

Author: vsoch

MANIFEST.in

@@ -1,4 +1,4 @@
 recursive-include singularity/templates *
 recursive-include singularity/static *
-recursive-include singularity/scripts *
+recursive-include singularity/build *
 recursive-include singularity/testing *

setup.py

@@ -7,7 +7,7 @@
     name="singularity",
 
     # Version number:
-    version="0.37",
+    version="0.39",
 
     # Application author details:
     author="Vanessa Sochat",
@@ -18,7 +18,6 @@
 
     # Data files
     include_package_data=True,
-
     zip_safe=False,
 
     # Details

singularity/build/google.py

@@ -11,8 +11,11 @@
     api_post
 )
 
+from singularity.build.utils import get_singularity_version
+
 from singularity.package import (
     build_from_spec, 
+    estimate_image_size,
     package
 )
 
@@ -59,7 +62,13 @@
 def get_storage_service():
     credentials = GoogleCredentials.get_application_default()
     return build('storage', 'v1', credentials=credentials)
+
+
+def get_compute_service():
+    credentials = GoogleCredentials.get_application_default()
+    return build('compute', 'v1', credentials=credentials)
 
+
 def get_bucket(storage_service,bucket_name):
     req = storage_service.buckets().get(bucket=bucket_name)
     return req.execute()
@@ -106,7 +115,7 @@ def list_bucket(bucket,storage_service):
 
 
 def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,bucket_name=None,
-              repo_id=None,commit=None,verbose=True,response_url=None,secret=None):
+              repo_id=None,commit=None,verbose=True,response_url=None,secret=None,branch=None):
     '''run_build will generate the Singularity build from a spec_file from a repo_url. 
     If no arguments are required, the metadata api is queried for the values.
     :param build_dir: directory to do the build in. If not specified,
@@ -115,13 +124,14 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
     :param repo_url: the url to download the repo from
     :param repo_id: the repo_id to uniquely identify the repo (in case name changes)
     :param commit: the commit to checkout. If none provided, will use most recent.
-    :param size: the size of the image to build. If none set, builds default 1024.
+    :param size: the size of the image to build. If none set, builds folder size + 200MB padding
     :param bucket_name: the name of the bucket to send files to
     :param verbose: print out extra details as we go (default True)    
     :param token: a token to send back to the server to authenticate the collection
     :param secret: a secret to match to the correct container
     :param response_url: the build url to send the response back to. Should also come
     from metadata. If not specified, no response is sent
+    :param branch: the branch to checkout for the build.
     :: note: this function is currently configured to work with Google Compute
     Engine metadata api, and should (will) be customized if needed to work elsewhere 
     '''
@@ -145,7 +155,9 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                 {'key': 'token', 'value': token, 'return_text': False },
                 {'key': 'commit', 'value': commit, 'return_text': True },
                 {'key': 'secret', 'value': secret, 'return_text': True },
-                {'key': 'size', 'value': size, 'return_text': True }]
+                {'key': 'size', 'value': size, 'return_text': True },
+                {'key': 'branch', 'value': branch, 'return_text': True },
+                {'key': 'container_id', 'value': None, 'return_text': True }]
 
     # Default spec file is Singularity
     if spec_file == None:
@@ -162,6 +174,10 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                          destination=build_dir)
 
     os.chdir(build_dir)
+    if params['branch'] != None:
+        bot.logger.info('Checking out branch %s',params['branch'])
+        os.system('git checkout %s' %(params['branch']))
+
     if params['commit'] != None:
         bot.logger.info('Checking out commit %s',params['commit'])
         os.system('git checkout %s .' %(params['commit']))
@@ -175,6 +191,12 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
     if os.path.exists(spec_file):
         bot.logger.info("Found spec file %s in repository",spec_file)
 
+        # If size is None, get from image + 200 padding
+        if params['size'] == None:
+            bot.logger.info("Size not detected for build. Will estimate with 200MB padding.")
+            params['size'] = estimate_image_size(spec=spec_file,
+                                                 sudopw='')
+
         image = build_from_spec(spec=spec_file, # default will package the image
                                 size=params['size'],
                                 sudopw='', # with root should not need sudo
@@ -232,7 +254,17 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                         "repo_url": params['repo_url'],
                         "commit": params['commit'],
                         "repo_id": params['repo_id'],
-                        "secret": params['secret']}
+                        "secret": params['secret'],
+                        "container_id": params['container_id']}
+
+            # Did the user specify a specific log file?
+            logfile = get_build_metadata(key='logfile')
+            if logfile != None:
+                response['logfile'] = logfile
+
+            if params['branch'] != None:
+                response['branch'] = params['branch']
+
 
             if params['token'] != None:
                 response['token'] = params['token']
@@ -252,7 +284,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
     shutil.rmtree(build_dir)
 
 
-def finish_build(logfile,repo_url=None,bucket_name=None,commit=None,verbose=True,repo_id=None,
+def finish_build(logfile,singularity_version=None,repo_url=None,bucket_name=None,commit=None,verbose=True,repo_id=None,
                  logging_response_url=None,secret=None,token=None):
     '''finish_build will finish the build by way of sending the log to the same bucket.
     :param build_dir: directory to do the build in. If not specified,
@@ -262,6 +294,7 @@ def finish_build(logfile,repo_url=None,bucket_name=None,commit=None,verbose=True
     :param repo_id: the repo_id to uniquely identify the repo (in case name changes)
     :param commit: the commit to checkout. If none provided, will use most recent.
     :param bucket_name: the name of the bucket to send files to
+    :param singularity_version: the version of singularity installed
     :param verbose: print out extra details as we go (default True)    
     :param secret: a secret to match to the correct container
     :param logging_response_url: the logging response url to send the response back to.
@@ -273,6 +306,9 @@ def finish_build(logfile,repo_url=None,bucket_name=None,commit=None,verbose=True
     if go == None:
         sys.exit(0)
 
+    if singularity_version == None:
+        singularity_version = get_singularity_version(singularity_version)
+
     # Get variables from the instance metadata API
     metadata = [{'key': 'logging_url', 'value': logging_response_url, 'return_text': True },
                 {'key': 'repo_url', 'value': repo_url, 'return_text': False },

singularity/build/scripts/singularity-build-latest.sh

@@ -5,7 +5,7 @@ sudo apt-get install -y build-essential libtool autotools-dev automake autoconf
 sudo apt-get install -y python3-pip
 cd /tmp && git clone http://www.github.com/singularityware/singularity
 cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
-SINGULARITY_VERSION=$(singularity --version)
+export SINGULARITY_VERSION=$(singularity --version)
 sudo pip3 install --upgrade pip
 sudo pip3 install --upgrade google-api-python-client
 sudo pip3 install --upgrade google
@@ -14,5 +14,5 @@ sudo pip3 install gitpython
 cd /tmp && git clone https://github.com/singularityware/singularity-python
 cd singularity-python && python3 setup.py sdist && sudo python3 setup.py install
 python3 -c "from singularity.build.google import run_build; run_build()" > /tmp/.shub-log 2>&1
-command=$(echo "from singularity.build.google import finish_build; finish_build(logfile='/tmp/.shub-log',version='$SINGULARITY_VERSION')")
+command=$(echo "from singularity.build.google import finish_build; finish_build(logfile='/tmp/.shub-log')")
 python3 -c "$command"

singularity/build/utils.py

@@ -14,7 +14,8 @@
 
 from singularity.utils import (
     get_installdir,
-    read_file
+    read_file,
+    run_command
 )
 
 import sys
@@ -59,3 +60,19 @@ def get_build_template(template_name,params=None,to_file=None):
     else:
         bot.logger.warning("Template %s not found.",template_file)
         return None
+
+
+def get_singularity_version(singularity_version=None):
+    '''get_singularity_version will determine the singularity version for a build
+    first, an environmental variable is looked at, followed by using the system
+    version.
+    '''
+    if singularity_version == None:        
+        singularity_version = os.environ.get("SINGULARITY_VERSION",None)
+        
+    # Next get from system
+    if singularity_version == None:
+        cmd = ['singularity','--version']
+        singularity_version = run_command(cmd,error_message="Cannot determine Singularity version!").decode('utf-8').strip('\n')
+
+    return singularity_version

singularity/package.py

@@ -8,6 +8,7 @@
 from singularity.logman import bot
 
 from singularity.utils import (
+    calculate_folder_size,
     format_container_name,
     read_file, 
     zip_up
@@ -23,10 +24,34 @@
 import os
 
 
+def estimate_image_size(spec=None,sudopw=None,padding=200):
+    '''estimate_image_size will generate an image in a directory, and add
+    some padding to it to estimate the size of the image file to generate
+    :param sudopw: the sudopw for Singularity, root should provide ''
+    :param spec: the spec file, called "Singuarity"
+    :param padding: the padding (MB) to add to the image
+    '''
+    size_dir = tempfile.mkdtemp()
+    tmp_dir = tempfile.mkdtemp()
+    image_folder = build_from_spec(spec=spec_file, # default will package the image
+                                   sudopw=sudopw, # with root should not need sudo
+                                   output_folder=size_dir,
+                                   build_dir=tmp_dir,
+                                   build_folder=True)
+    original_size = calculate_folder_size(image_folder)    
+    bot.logger.debug("Original image size calculated as %s",original_size)
+    padded_size = original_size + padding
+    bot.logger.debug("Size with padding will be %s",padded_size)
+    shutil.rmtree(size_dir)
+    os.system('sudo rm -rf %s' %tmp_dir)
+    return padded_size
+
+
 def build_from_spec(spec=None,build_dir=None,size=None,sudopw=None,
                     output_folder=None,build_folder=False):
     '''build_from_spec will build a "spec" file in a "build_dir" and return the directory
     :param spec: the spec file, called "Singuarity"
+    :param sudopw: the sudopw for Singularity, root should provide ''
     :param build_dir: the directory to build in. If not defined, will use tmpdir.
     :param size: the size of the image
     :param output_folder: where to output the image package

singularity/tests/test_package.py

@@ -62,5 +62,6 @@ def test_packaging(self):
         self.assertTrue(os.path.exists(image1_extraction))
         shutil.rmtree(os.path.dirname(image1_extraction))
 
+
 if __name__ == '__main__':
     unittest.main()

singularity/utils.py

@@ -39,13 +39,14 @@
 ######################################################################################
 
 
-def check_install(software="singularity"):
+def check_install(software=None):
     '''check_install will attempt to run the singularity command, and return an error
     if not installed. The command line utils will not run without this check.
     '''    
-  
+    if software == None:
+        software = "singularity"
     cmd = [software,'--version']
-    version = run_command(cmd,error_message="Cannot find singularity. Is it installed?")
+    version = run_command(cmd,error_message="Cannot find %s. Is it installed?" %software)
     if version != None:
         bot.logger.info("Found %s version %s",software.upper(),version)
         return True
@@ -228,6 +229,24 @@ def read_file(filename,mode="r"):
     return content
 
 
+############################################################################
+## OTHER MISC. #############################################################
+############################################################################
+
+
+def calculate_folder_size(folder_path):
+    '''calculate_folder size recursively walks a directory to calculate
+    a total size (in)
+    :param folder_path: the path to calculate size for
+    '''
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(folder_path):
+        for filey in filenames:
+            fp = os.path.join(dirpath, filey)
+            total_size += os.path.getsize(fp)
+    return total_size
+
+
 def remove_unicode_dict(input_dict):
     '''remove unicode keys and values from dict, encoding in utf8
     '''