Merge pull request #40 from vsoch/master

tweaks for builder

Author: vsoch

setup.py

@@ -7,7 +7,7 @@
     name="singularity",
 
     # Version number:
-    version="0.39",
+    version="0.44",
 
     # Application author details:
     author="Vanessa Sochat",

singularity/build/google.py

@@ -47,6 +47,7 @@
 import subprocess
 import sys
 import tempfile
+import time
 import uuid
 import zipfile
 
@@ -157,18 +158,18 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                 {'key': 'secret', 'value': secret, 'return_text': True },
                 {'key': 'size', 'value': size, 'return_text': True },
                 {'key': 'branch', 'value': branch, 'return_text': True },
-                {'key': 'container_id', 'value': None, 'return_text': True }]
-
-    # Default spec file is Singularity
-    if spec_file == None:
-        spec_file = "Singularity"
-
-    if bucket_name == None:
-        bucket_name = "singularity-hub"
+                {'key': 'spec_file', 'value': spec_file, 'return_text': True }]
 
     # Obtain values from build
     params = get_build_params(metadata)
 
+    # Default spec file is Singularity
+    if params['spec_file'] == None:
+        params['spec_file'] = "Singularity"
+
+    if params['bucket_name'] == None:
+        params['bucket_name'] = "singularity-hub"
+
     # Download the repo and image
     repo = download_repo(repo_url=params['repo_url'],
                          destination=build_dir)
@@ -188,16 +189,16 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
         bot.logger.warning("commit not specified, setting to current %s", params['commit'])
 
 
-    if os.path.exists(spec_file):
-        bot.logger.info("Found spec file %s in repository",spec_file)
+    if os.path.exists(params['spec_file']):
+        bot.logger.info("Found spec file %s in repository",params['spec_file'])
 
-        # If size is None, get from image + 200 padding
+        # If size is None, get from image + 50 padding
         if params['size'] == None:
             bot.logger.info("Size not detected for build. Will estimate with 200MB padding.")
-            params['size'] = estimate_image_size(spec=spec_file,
+            params['size'] = estimate_image_size(spec_file=os.path.abspath(params['spec_file']),
                                                  sudopw='')
 
-        image = build_from_spec(spec=spec_file, # default will package the image
+        image = build_from_spec(spec_file=params['spec_file'], # default will package the image
                                 size=params['size'],
                                 sudopw='', # with root should not need sudo
                                 output_folder=build_dir,
@@ -209,7 +210,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
 
         # Package the image metadata (files, folders, etc)
         image_package = package(image_path=image,
-                                spec_path=spec_file,
+                                spec_path=params['spec_file'],
                                 output_folder=build_dir,
                                 sudopw='',
                                 remove_image=True,
@@ -254,8 +255,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
                         "repo_url": params['repo_url'],
                         "commit": params['commit'],
                         "repo_id": params['repo_id'],
-                        "secret": params['secret'],
-                        "container_id": params['container_id']}
+                        "secret": params['secret']}
 
             # Did the user specify a specific log file?
             logfile = get_build_metadata(key='logfile')
@@ -276,16 +276,16 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
     else:
         # Tell the user what is actually there
         present_files = glob("*")
-        bot.logger.error("Build file %s not found in repository",spec_file)
+        bot.logger.error("Build file %s not found in repository",params['spec_file'])
         bot.logger.info("Found files are %s","\n".join(present_files))
 
 
     # Clean up
     shutil.rmtree(build_dir)
 
 
-def finish_build(logfile,singularity_version=None,repo_url=None,bucket_name=None,commit=None,verbose=True,repo_id=None,
-                 logging_response_url=None,secret=None,token=None):
+def finish_build(logfile=None,singularity_version=None,repo_url=None,bucket_name=None,commit=None,
+                 logging_url=None,secret=None,token=None,verbose=True,repo_id=None):
     '''finish_build will finish the build by way of sending the log to the same bucket.
     :param build_dir: directory to do the build in. If not specified,
     will use temporary.   
@@ -297,7 +297,7 @@ def finish_build(logfile,singularity_version=None,repo_url=None,bucket_name=None
     :param singularity_version: the version of singularity installed
     :param verbose: print out extra details as we go (default True)    
     :param secret: a secret to match to the correct container
-    :param logging_response_url: the logging response url to send the response back to.
+    :param logging_url: the logging response url to send the response back to.
     :: note: this function is currently configured to work with Google Compute
     Engine metadata api, and should (will) be customized if needed to work elsewhere 
     '''
@@ -310,13 +310,14 @@ def finish_build(logfile,singularity_version=None,repo_url=None,bucket_name=None
         singularity_version = get_singularity_version(singularity_version)
 
     # Get variables from the instance metadata API
-    metadata = [{'key': 'logging_url', 'value': logging_response_url, 'return_text': True },
+    metadata = [{'key': 'logging_url', 'value': logging_url, 'return_text': True },
                 {'key': 'repo_url', 'value': repo_url, 'return_text': False },
                 {'key': 'repo_id', 'value': repo_id, 'return_text': True },
                 {'key': 'token', 'value': token, 'return_text': False },
                 {'key': 'commit', 'value': commit, 'return_text': True },
                 {'key': 'bucket_name', 'value': bucket_name, 'return_text': True },
-                {'key': 'secret', 'value': secret, 'return_text': True }]
+                {'key': 'secret', 'value': secret, 'return_text': True }, 
+                {'key': 'logfile', 'value': logfile, 'return_text': True }]
 
     if bucket_name == None:
         bucket_name = "singularity-hub"
@@ -333,24 +334,28 @@ def finish_build(logfile,singularity_version=None,repo_url=None,bucket_name=None
     log_file = upload_file(storage_service,
                            bucket=bucket,
                            bucket_path=image_path,
-                           file_name=logfile)
+                           file_name=params['logfile'])
 
     # Finally, package everything to send back to shub
     response = {"log": json.dumps(log_file),
                 "repo_url": params['repo_url'],
                 "commit": params['commit'],
+                "logfile": params['logfile'],
                 "repo_id": params['repo_id'],
                 "secret": params['secret']}
 
-
     if params['token'] != None:
         response['token'] = params['token']
 
     # Send it back!
     if params['logging_url'] != None:
         finish = requests.post(params['logging_url'],data=response)
+   
+    # Delay a minute, to give buffer between bringing instance down
+    time.sleep(60)
 
 
+
 #####################################################################################
 # METADATA
 #####################################################################################
@@ -393,7 +398,7 @@ def get_build_params(metadata):
             bot.logger.warning('%s not found in function call.',item['key'])        
             response = get_build_metadata(key=item['key'])
             item['value'] = response
-            params[item['key']] = item['value']
+        params[item['key']] = item['value']
         if item['key'] != 'credential':
             bot.logger.info('%s is set to %s',item['key'],item['value'])        
     return params

singularity/build/scripts/singularity-build-latest.sh

@@ -12,7 +12,7 @@ sudo pip3 install --upgrade google
 sudo pip3 install oauth2client==3.0.0
 sudo pip3 install gitpython
 cd /tmp && git clone https://github.com/singularityware/singularity-python
-cd singularity-python && python3 setup.py sdist && sudo python3 setup.py install
+cd singularity-python && python3 setup.py sdist && sudo python3 setup.py install && cd ..
 python3 -c "from singularity.build.google import run_build; run_build()" > /tmp/.shub-log 2>&1
-command=$(echo "from singularity.build.google import finish_build; finish_build(logfile='/tmp/.shub-log')")
+command=$(echo "from singularity.build.google import finish_build; finish_build()")
 python3 -c "$command"

singularity/package.py

@@ -24,16 +24,16 @@
 import os
 
 
-def estimate_image_size(spec=None,sudopw=None,padding=200):
+def estimate_image_size(spec_file,sudopw=None,padding=50):
     '''estimate_image_size will generate an image in a directory, and add
     some padding to it to estimate the size of the image file to generate
     :param sudopw: the sudopw for Singularity, root should provide ''
-    :param spec: the spec file, called "Singuarity"
+    :param spec_file: the spec file, called "Singuarity"
     :param padding: the padding (MB) to add to the image
     '''
     size_dir = tempfile.mkdtemp()
     tmp_dir = tempfile.mkdtemp()
-    image_folder = build_from_spec(spec=spec_file, # default will package the image
+    image_folder = build_from_spec(spec_file=spec_file, # default will package the image
                                    sudopw=sudopw, # with root should not need sudo
                                    output_folder=size_dir,
                                    build_dir=tmp_dir,
@@ -47,26 +47,26 @@ def estimate_image_size(spec=None,sudopw=None,padding=200):
     return padded_size
 
 
-def build_from_spec(spec=None,build_dir=None,size=None,sudopw=None,
+def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,
                     output_folder=None,build_folder=False):
     '''build_from_spec will build a "spec" file in a "build_dir" and return the directory
-    :param spec: the spec file, called "Singuarity"
+    :param spec_file: the spec file, called "Singuarity"
     :param sudopw: the sudopw for Singularity, root should provide ''
     :param build_dir: the directory to build in. If not defined, will use tmpdir.
     :param size: the size of the image
     :param output_folder: where to output the image package
     :param build_folder: "build" the image into a folder instead. Default False
     '''
-    if spec == None:
-        spec = "Singularity"
+    if spec_file == None:
+        spec_file = "Singularity"
     if build_dir == None:
         build_dir = tempfile.mkdtemp()
     bot.logger.debug("Building in directory %s",build_dir)
 
     # Copy the spec to a temporary directory
-    spec_path = "%s/%s" %(build_dir,spec)
+    spec_path = "%s/%s" %(build_dir,spec_file)
     if not os.path.exists(spec_path):
-        shutil.copyfile(spec,spec_path)
+        shutil.copyfile(spec_file,spec_path)
     # If name isn't provided, call it Singularity
     image_path = "%s/image" %(build_dir)
     # Run create image and bootstrap with Singularity command line tool.

singularity/utils.py

@@ -234,17 +234,22 @@ def read_file(filename,mode="r"):
 ############################################################################
 
 
-def calculate_folder_size(folder_path):
+def calculate_folder_size(folder_path,truncate=True):
     '''calculate_folder size recursively walks a directory to calculate
-    a total size (in)
+    a total size (in MB)
     :param folder_path: the path to calculate size for
+    :param truncate: if True, converts size to an int
     '''
     total_size = 0
     for dirpath, dirnames, filenames in os.walk(folder_path):
         for filey in filenames:
-            fp = os.path.join(dirpath, filey)
-            total_size += os.path.getsize(fp)
-    return total_size
+            file_path = os.path.join(dirpath, filey)
+            if os.path.isfile(file_path) and not os.path.islink(file_path):
+                total_size += os.path.getsize(file_path) # this is bytes
+    size_mb = total_size / 1000000
+    if truncate == True:
+        size_mb = int(size_mb)
+    return size_mb
 
 
 def remove_unicode_dict(input_dict):