Merge pull request #36 from vsoch/master

adding build specs to singularity python to close #35

Author: vsoch

.travis.yml

@@ -14,8 +14,10 @@ install:
   - cd $TRAVIS_BUILD_DIR/
   - cd $TRAVIS_BUILD_DIR && pip install -r requirements.txt
   - cd $TRAVIS_BUILD_DIR && pip3 install -r requirements.txt
+  - python setup.py sdist
+  - python setup.py install
   - pylint --version
-#  - cd $TRAVIS_BUILD_DIR/libexec/python && pylint $PWD --errors-only --ignore tests  --disable=E0401,E0611
 
 script:
-  - bash $TRAVIS_BUILD_DIR/singularity/testing/run_tests.sh $TRAVIS_BUILD_DIR/singularity/testing /tmp
+  - python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'
+  #bash $TRAVIS_BUILD_DIR/singularity/tests/run_tests.sh $TRAVIS_BUILD_DIR/singularity/tests /tmp

setup.py

@@ -15,9 +15,10 @@
 
     # Packages
     packages=find_packages(),
-
+ 
     # Data files
     include_package_data=True,
+
     zip_safe=False,
 
     # Details

singularity/api.py

@@ -73,7 +73,7 @@ def api_get(url,headers=None,token=None,data=None):
     :param headers: a dictionary with headers for the request
     :param putdata: additional data to add to the request
     '''
-    bot.logger.debug("GET ",url)
+    bot.logger.debug("GET %s",url)
 
     if headers == None:
         headers = get_headers(token=token)
@@ -94,7 +94,7 @@ def api_put(url,headers=None,token=None,data=None):
     :param headers: a dictionary with headers for the request
     :param data: additional data to add to the request
     '''
-    bot.logger.debug("PUT ",url)
+    bot.logger.debug("PUT %s",url)
 
     if headers == None:
         headers = get_headers(token=token)
@@ -115,7 +115,7 @@ def api_post(url,headers=None,data=None,token=None):
     :param headers: a dictionary with headers for the request
     :param data: additional data to add to the request
     '''
-    bot.logger.debug("POST ",url)
+    bot.logger.debug("POST %s",url)
 
     if headers == None:
         headers = get_headers(token=token)

singularity/testing/__init__.py singularity/build/__init__.pysingularity/testing/__init__.py renamed to singularity/build/__init__.py

@@ -1,13 +1,27 @@
 #!/usr/bin/env python
 
 '''
-build.py: functions for singularity hub builders
+build/google.py: build functions for singularity hub google compute engine
 
 '''
 
-from singularity.api import api_get, api_put, api_post
-from singularity.package import build_from_spec, package
-from singularity.utils import get_installdir, read_file, write_file, download_repo
+from singularity.api import (
+    api_get, 
+    api_put, 
+    api_post
+)
+
+from singularity.package import (
+    build_from_spec, 
+    package
+)
+
+from singularity.utils import (
+    get_installdir, 
+    read_file, 
+    write_file, 
+    download_repo
+)
 
 from googleapiclient.discovery import build
 from oauth2client.client import GoogleCredentials
@@ -79,7 +93,7 @@ def upload_file(storage_service,bucket,bucket_path,file_name,verbose=True):
     return request.execute()
 
 
-def list_bucket(bucket):
+def list_bucket(bucket,storage_service):
     # Create a request to objects.list to retrieve a list of objects.        
     request = storage_service.objects().list(bucket=bucket['id'], 
                                              fields='nextPageToken,items(name,size,contentType)')

singularity/build.py singularity/build/google.pysingularity/build.py renamed to singularity/build/google.py

@@ -0,0 +1,18 @@
+#!/bin/sh
+sudo apt-get update
+sudo apt-get -y install git
+sudo apt-get install -y build-essential libtool autotools-dev automake autoconf
+sudo apt-get install -y python3-pip
+cd /tmp && git clone http://www.github.com/singularityware/singularity
+cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+SINGULARITY_VERSION=$(singularity --version)
+sudo pip3 install --upgrade pip
+sudo pip3 install --upgrade google-api-python-client
+sudo pip3 install --upgrade google
+sudo pip3 install oauth2client==3.0.0
+sudo pip3 install gitpython
+cd /tmp && git clone https://github.com/singularityware/singularity-python
+cd singularity-python && python3 setup.py sdist && sudo python3 setup.py install
+python3 -c "from singularity.build.google import run_build; run_build()" > /tmp/.shub-log 2>&1
+command=$(echo "from singularity.build.google import finish_build; finish_build(logfile='/tmp/.shub-log',version='$SINGULARITY_VERSION')")
+python3 -c "$command"

singularity/build/scripts/singularity-build-latest.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+'''
+build/utils.py: general building util functions
+
+'''
+
+import os
+import re
+
+import shutil
+import simplejson
+from singularity.logman import bot
+
+from singularity.utils import (
+    get_installdir,
+    read_file
+)
+
+import sys
+
+import subprocess
+
+import tempfile
+import zipfile
+
+
+######################################################################################
+# Build Templates
+######################################################################################
+
+def get_build_template(template_name,params=None,to_file=None):
+    '''get_build template returns a string or file for a particular build template, which is
+    intended to build a version of a Singularity image on a cloud resource.
+    :param template_name: the name of the template to retrieve in build/scripts
+    :param params: (if needed) a dictionary of parameters to substitute in the file
+    :param to_file: if defined, will write to file. Default returns string.
+    '''
+    base = get_installdir()
+    template_folder = "%s/build/scripts" %(base)
+    template_file = "%s/%s" %(template_folder,template_name)
+    if os.path.exists(template_file):
+        bot.logger.debug("Found template %s",template_file)
+
+        # Implement when needed - substitute params here
+        # Will need to read in file instead of copying below
+        # if params != None:
+ 
+        if to_file != None:
+            shutil.copyfile(template_file,to_file)
+            bot.logger.debug("Template file saved to %s",to_file)
+            return to_file
+
+        # If the user wants a string
+        content = ''.join(read_file(template_file)) 
+        return content
+
+
+    else:
+        bot.logger.warning("Template %s not found.",template_file)
+        return None

singularity/build/utils.py

@@ -85,16 +85,9 @@ def get_runscript_parameters(runscript,name,version,description=None):
 
     if language == 'python':
         params = get_parameters_python(runscript)
+        # TODO: here we would write something like boutiqes spec
 
-    # Finally, make a boutiques json object with the parameters
-    if params != None:
-        json_spec = get_boutiques_json(name=name,
-                                       version=version,
-                                       inputs=params,
-                                       command=name,
-                                       description=description)
-
-    return json_spec
+    return params
 
 
 def get_parameters_python(runscript):