Merge pull request #41 from vsoch/master

need sleep

Author: vsoch

setup.py

@@ -7,7 +7,7 @@
     name="singularity",
 
     # Version number:
-    version="0.44",
+    version="0.59",
 
     # Application author details:
     author="Vanessa Sochat",

singularity/build/google.py

@@ -194,14 +194,14 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
 
         # If size is None, get from image + 50 padding
         if params['size'] == None:
-            bot.logger.info("Size not detected for build. Will estimate with 200MB padding.")
-            params['size'] = estimate_image_size(spec_file=os.path.abspath(params['spec_file']),
-                                                 sudopw='')
+            bot.logger.info("Size not detected for build. Will estimate as 1024MB.")
+            #params['size'] = estimate_image_size(spec_file=os.path.abspath(params['spec_file']),
+            #                                     sudopw='')
+            params['size'] = 1024
 
         image = build_from_spec(spec_file=params['spec_file'], # default will package the image
                                 size=params['size'],
                                 sudopw='', # with root should not need sudo
-                                output_folder=build_dir,
                                 build_dir=build_dir)
 
         # Compress image
@@ -232,7 +232,7 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
 
             # Start the storage service, retrieve the bucket
             storage_service = get_storage_service()
-            bucket = get_bucket(storage_service,bucket_name)
+            bucket = get_bucket(storage_service,params["bucket_name"])
 
             # For each file, upload to storage
             files = []
@@ -283,6 +283,9 @@ def run_build(build_dir=None,spec_file=None,repo_url=None,token=None,size=None,b
     # Clean up
     shutil.rmtree(build_dir)
 
+    # Delay a minute, to give buffer between bringing instance down
+    time.sleep(60)
+
 
 def finish_build(logfile=None,singularity_version=None,repo_url=None,bucket_name=None,commit=None,
                  logging_url=None,secret=None,token=None,verbose=True,repo_id=None):
@@ -339,7 +342,6 @@ def finish_build(logfile=None,singularity_version=None,repo_url=None,bucket_name
     # Finally, package everything to send back to shub
     response = {"log": json.dumps(log_file),
                 "repo_url": params['repo_url'],
-                "commit": params['commit'],
                 "logfile": params['logfile'],
                 "repo_id": params['repo_id'],
                 "secret": params['secret']}
@@ -350,10 +352,7 @@ def finish_build(logfile=None,singularity_version=None,repo_url=None,bucket_name
     # Send it back!
     if params['logging_url'] != None:
         finish = requests.post(params['logging_url'],data=response)
-   
-    # Delay a minute, to give buffer between bringing instance down
-    time.sleep(60)
-    
+       
 
 
 #####################################################################################

singularity/build/scripts/singularity-build-latest.sh

@@ -1,18 +1,26 @@
 #!/bin/sh
-sudo apt-get update
-sudo apt-get -y install git
-sudo apt-get install -y build-essential libtool autotools-dev automake autoconf
-sudo apt-get install -y python3-pip
-cd /tmp && git clone http://www.github.com/singularityware/singularity
-cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
-export SINGULARITY_VERSION=$(singularity --version)
-sudo pip3 install --upgrade pip
-sudo pip3 install --upgrade google-api-python-client
-sudo pip3 install --upgrade google
-sudo pip3 install oauth2client==3.0.0
-sudo pip3 install gitpython
-cd /tmp && git clone https://github.com/singularityware/singularity-python
-cd singularity-python && python3 setup.py sdist && sudo python3 setup.py install && cd ..
+sudo apt-get update &&
+sudo apt-get -y install git \
+                        build-essential \
+                        libtool \
+                        autotools-dev \
+                        automake \
+                        autoconf \
+                        python3-pip
+
+# Install Singularity from Github
+cd /tmp && git clone http://www.github.com/singularityware/singularity &&
+   cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+
+# Pip3 installs
+sudo pip3 install --upgrade pip &&
+sudo pip3 install --upgrade google-api-python-client &&
+sudo pip3 install --upgrade google &&
+sudo pip3 install oauth2client==3.0.0 gitpython singularity
+
+# Main running script
 python3 -c "from singularity.build.google import run_build; run_build()" > /tmp/.shub-log 2>&1
-command=$(echo "from singularity.build.google import finish_build; finish_build()")
+
+# Finish by sending log
+export command=$(echo "from singularity.build.google import finish_build; finish_build()") &&
 python3 -c "$command"

singularity/build/utils.py

@@ -72,7 +72,12 @@ def get_singularity_version(singularity_version=None):
 
     # Next get from system
     if singularity_version == None:
-        cmd = ['singularity','--version']
-        singularity_version = run_command(cmd,error_message="Cannot determine Singularity version!").decode('utf-8').strip('\n')
+        try:
+            cmd = ['singularity','--version']
+            singularity_version = run_command(cmd,error_message="Cannot determine Singularity version!").decode('utf-8').strip('\n')
+            bot.logger.info("Singularity %s being used.",singularity_version)
+        except:
+            singularity_version = None
+            bot.logger.warning("Singularity version not found, so it's likely not installed.")
 
     return singularity_version

singularity/package.py

@@ -31,63 +31,68 @@ def estimate_image_size(spec_file,sudopw=None,padding=50):
     :param spec_file: the spec file, called "Singuarity"
     :param padding: the padding (MB) to add to the image
     '''
-    size_dir = tempfile.mkdtemp()
-    tmp_dir = tempfile.mkdtemp()
     image_folder = build_from_spec(spec_file=spec_file, # default will package the image
                                    sudopw=sudopw, # with root should not need sudo
-                                   output_folder=size_dir,
-                                   build_dir=tmp_dir,
                                    build_folder=True)
     original_size = calculate_folder_size(image_folder)    
     bot.logger.debug("Original image size calculated as %s",original_size)
     padded_size = original_size + padding
     bot.logger.debug("Size with padding will be %s",padded_size)
-    shutil.rmtree(size_dir)
-    os.system('sudo rm -rf %s' %tmp_dir)
     return padded_size
 
 
-def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,
-                    output_folder=None,build_folder=False):
+def build_from_spec(spec_file=None,build_dir=None,size=None,sudopw=None,build_folder=False):
     '''build_from_spec will build a "spec" file in a "build_dir" and return the directory
     :param spec_file: the spec file, called "Singuarity"
     :param sudopw: the sudopw for Singularity, root should provide ''
     :param build_dir: the directory to build in. If not defined, will use tmpdir.
     :param size: the size of the image
-    :param output_folder: where to output the image package
     :param build_folder: "build" the image into a folder instead. Default False
     '''
     if spec_file == None:
         spec_file = "Singularity"
+
     if build_dir == None:
         build_dir = tempfile.mkdtemp()
+
     bot.logger.debug("Building in directory %s",build_dir)
 
     # Copy the spec to a temporary directory
-    spec_path = "%s/%s" %(build_dir,spec_file)
+    bot.logger.debug("Spec file set to %s",spec_file)
+    spec_path = "%s/%s" %(build_dir,os.path.basename(spec_file))
+    bot.logger.debug("Spec file for build should be in %s",spec_path)
+
+    # If it's not already there
     if not os.path.exists(spec_path):
         shutil.copyfile(spec_file,spec_path)
-    # If name isn't provided, call it Singularity
+
     image_path = "%s/image" %(build_dir)
+
     # Run create image and bootstrap with Singularity command line tool.
     if sudopw != None:
         cli = Singularity(sudopw=sudopw)
     else:
         cli = Singularity() # This command will ask the user for sudo
+
     print("\nCreating and boostrapping image...")
+
     # Does the user want to "build" into a folder or image?
     if build_folder == True:
+        bot.logger.debug("build_folder is true, creating %s",image_path)
         os.mkdir(image_path)
     else:
         cli.create(image_path,size=size)
+
     result = cli.bootstrap(image_path=image_path,spec_path=spec_path)
     print(result)
+
     # If image, rename based on hash
     if build_folder == False:
         version = get_image_hash(image_path)
         final_path = "%s/%s" %(build_dir,version)
         os.rename(image_path,final_path)
         image_path = final_path
+
     bot.logger.debug("Built image: %s",image_path)
     return image_path