Merge pull request #80 from vsoch/development

vsoch · web-flow · commit 1b38684fcda7 · 2017-10-31T19:38:18.000-04:00
Development
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,20 @@
+# CHANGELOG
+
+This is a manually generated log to track changes to the repository for each release. 
+Each section should include general headers such as **Implemented enhancements** 
+and **Merged pull requests**. All closed issued and bug fixes should be 
+represented by the pull requests that fixed them. This log originated with Singularity 2.4
+and changes prior to that are (unfortunately) done retrospectively. Critical items to know are:
+
+ - renamed commands
+ - deprecated / removed commands
+ - changed defaults
+ - backward incompatible changes (recipe file format? image file format?)
+ - migration guidance (how to convert images?)
+ - changed behaviour (recipe sections work differently)
+
+
+## [vxx](https://github.com/singularityware/singularity-python/tree/development) (development)
+
+**changed defaults**
+ - *sregistry client*: to support use of squashfs images and singularity 2.4, the default upload is not compressed, assuming squashfs, and the default download is not decompressed. To still compress an image add the `--compress` flag on push, and the `--decompress` flag on pull.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,3 +3,5 @@ recursive-include singularity *
 recursive-exclude * __pycache__
 recursive-exclude * *.pyc
 recursive-exclude * *.pyo
+recursive-exclude * *.simg
+recursive-exclude * *.img
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 
 Singularity Python is a python module and command line tool for working with <a href="https://singularityware.github.io" target="_blank">Singularity</a> containers, specifically providing functions to visualize, package, and compare containers. Currently, most functions use Singularity on the command line, and some require sudo.
 
-We currently require Python > version 3 to use various timezone functions. If you are unable to install version 3.0, we provide a [Singularity.container](Singularity.container) for you to use instead. See the [installation docs](https://github.com/singularityware/singularity-python/wiki/Installation) for your different options.
+We currently require Python > version 3 to use various timezone functions. If you are unable to install version 3.0, we provide a [Singularity.container](Singularity.container) for you to use instead. This is the recommended approach as some older versions of Python do not support generation of the timestamp. See the [installation docs](https://github.com/singularityware/singularity-python/wiki/Installation) for your different options.
 
 The Singularity-Python code is licensed under the MIT open source license, which is a highly permissive license that places few limits upon reuse. This ensures that the code will be usable by the greatest number of researchers, in both academia and industry. The following examples are provided:
 
diff --git a/singularity/analysis/classify.py b/singularity/analysis/classify.py
@@ -74,7 +74,7 @@ def get_diff(container=None,image_package=None):
     '''
     
     # Find the most similar os
-    most_similar = estimate_os(image_package=image_package,container=container) 
+    most_similar = estimate_os(image_package=image_package,container=container)
     similar_package = "%s/docker-os/%s.img.zip" %(get_package_base(),most_similar)
 
     comparison = compare_containers(image_package1=image_package,
diff --git a/singularity/analysis/compare.py b/singularity/analysis/compare.py
@@ -128,13 +128,13 @@ def compare_singularity_images(image_paths1,image_paths2=None):
                     members2 = [x.name for x in tar2]
                     c = compare_lists(members1,members2)
                     sim = information_coefficient(c['total1'],c['total2'],c['intersect'])
-                    delete_image_tar(fileobj2)
+                    delete_image_tar(fileobj2, tar2)
                         
                 dfs.loc[image1,image2] = sim
                 if repeat:
                     dfs.loc[image2,image1] = sim
                 comparisons_done.append(comparison_id)
-        delete_image_tar(fileobj1)
+        delete_image_tar(fileobj1, tar1)
     return dfs
 
 
diff --git a/singularity/analysis/reproduce/hash.py b/singularity/analysis/reproduce/hash.py
@@ -99,7 +99,7 @@ def get_image_hash(image_path,
                                 include_files=include_files)
                 
     cli = Singularity()
-    file_obj,tar = get_image_tar(image_path)
+    file_obj, tar = get_image_tar(image_path)
     hasher = hashlib.md5()
 
     for member in tar:
@@ -117,11 +117,14 @@ def get_image_hash(image_path,
 
     digest = hasher.hexdigest()
 
-    if isinstance(file_obj,io.BytesIO):
+    # Close up / remove files
+    try:
         file_obj.close()
-    else:
-        if os.path.exists(file_obj):
-            os.remove(file_obj)
+    except:
+        tar.close()
+ 
+    if os.path.exists(file_obj):
+        os.remove(file_obj)
 
     return digest
 
@@ -163,7 +166,7 @@ def get_content_hashes(image_path,
                            tag_root=tag_root,
                            include_sizes=include_sizes)
 
-    deleted = delete_image_tar(file_obj)
+    delete_image_tar(file_obj, tar)
     return results
 
 
diff --git a/singularity/analysis/reproduce/utils.py b/singularity/analysis/reproduce/utils.py
@@ -114,19 +114,17 @@ def get_image_tar(image_path,S=None):
     return file_obj, tar
 
 
-def delete_image_tar(file_obj):
+def delete_image_tar(file_obj, tar):
     '''delete image tar will close a file object (if extracted into
     memory) or delete from the file system (if saved to disk)'''
-    deleted = False
-    if isinstance(file_obj,io.BytesIO):
+    try:
         file_obj.close()
+    except:
+        tar.close()
+    if os.path.exists(file_obj):
+        os.remove(file_obj)
         deleted = True
-        bot.debug('Closed memory tar.')   
-    else:
-        if os.path.exists(file_obj):
-            os.remove(file_obj)
-            deleted = True
-            bot.debug('Deleted temporary tar.')   
+        bot.debug('Deleted temporary tar.')   
     return deleted
 
 
diff --git a/singularity/analysis/utils.py b/singularity/analysis/utils.py
@@ -32,7 +32,6 @@
 import requests
 
 import shutil
-import simplejson
 from singularity.logger import bot
 from singularity.utils import get_installdir
 import sys
diff --git a/singularity/build/google.py b/singularity/build/google.py
@@ -165,7 +165,7 @@ def get_image_path(repo_url, trailing_path):
 
 
 
-def run_build(logfile=None):
+def run_build(logfile='/tmp/.shub-log'):
 
     '''run_build will generate the Singularity build from a spec_file from a repo_url.
 
@@ -287,9 +287,10 @@ def run_build(logfile=None):
                     "metadata": json.dumps(metadata)}
 
         # Did the user specify a specific log file?
-        logfile = get_build_metadata('logfile')
-        if logfile is not None:
-            response['logfile'] = logfile
+        custom_logfile = get_build_metadata('logfile')
+        if custom_logfile is not None:
+            logfile = custom_logfile    
+        response['logfile'] = logfile
 
         # Send final build data to instance
         send_build_data(build_dir=build_dir,
diff --git a/singularity/build/main.py b/singularity/build/main.py
@@ -121,6 +121,11 @@ def run_build(build_dir,params,verbose=True, compress_image=False):
     if os.path.exists(params['spec_file']):
         bot.info("Found spec file %s in repository" %params['spec_file'])
 
+        # If the user has a symbolic link
+        if os.path.islink(params['spec_file']):
+            bot.info("%s is a symbolic link." %params['spec_file'])
+            params['spec_file'] = os.path.realpath(params['spec_file'])
+
         # START TIMING
         start_time = datetime.now()
         image = build_from_spec(spec_file=params['spec_file'], # default will package the image
diff --git a/singularity/build/utils.py b/singularity/build/utils.py
@@ -31,7 +31,6 @@
 import re
 
 import shutil
-import simplejson
 from singularity.logger import bot
 
 from singularity.utils import (
diff --git a/singularity/cli/__init__.py b/singularity/cli/__init__.py
@@ -61,7 +61,10 @@ def run_command(self, cmd, sudo=False, quiet=False):
                 try:
                     message=message.decode('utf-8')
                 except UnicodeDecodeError:
-                    message = unicode(message, errors='replace')
+                    try:
+                        message = unicode(message, errors='replace')
+                    except NameError:
+                        message = str(message, errors='replace')
             return message
         if quiet is False:
             bot.error("Return Code %s: %s" %(return_code,
diff --git a/singularity/package/build.py b/singularity/package/build.py
@@ -164,7 +164,7 @@ def package(image_path,
     zipfile = zip_up(to_package,zip_name=zip_name,output_folder=output_folder)
     bot.debug("Package created at %s" %(zipfile))
 
-    if not delete_image_tar(file_obj):
+    if not delete_image_tar(file_obj, tar):
         bot.warning("Could not clean up temporary tarfile.")
     
     # return package to user
diff --git a/singularity/package/utils.py b/singularity/package/utils.py
@@ -32,7 +32,6 @@
 
 import shutil
 import json
-import simplejson
 from singularity.logger import bot
 from singularity.utils import (
     write_json,
diff --git a/singularity/registry/auth.py b/singularity/registry/auth.py
@@ -61,8 +61,7 @@ def generate_signature(payload, secret):
 
 
 def generate_timestamp():
-    ts = datetime.now()
-    ts = ts.replace(tzinfo=timezone.utc)
+    ts = datetime.now(timezone.utc)
     return ts.strftime('%Y%m%dT%HZ')
 
 
diff --git a/singularity/registry/client/push.py b/singularity/registry/client/push.py
@@ -50,7 +50,7 @@
 )
 
 
-def push(self, path, name, tag=None, compress=True):
+def push(self, path, name, tag=None, compress=False):
     '''push an image to Singularity Registry'''
 
     path = os.path.abspath(path)
diff --git a/singularity/registry/main/__init__.py b/singularity/registry/main/__init__.py
@@ -97,8 +97,8 @@ def get_parser():
                        help='name of image, in format "library/image"', 
                        type=str, required=True)
 
-    push.add_argument('--no-compress', dest="nocompress", 
-                      help="show the test for each container.", 
+    push.add_argument('--compress', dest="compress", 
+                      help="compress the container on upload (for legacy containers prior to 2.4)", 
                       default=False, action='store_true')
 
 
@@ -114,8 +114,8 @@ def get_parser():
                        help='custom name for image', 
                        type=str, default=None)
 
-    pull.add_argument('--no-decompress', dest="nodecompress", 
-                      help="extract a gzip compressed image", 
+    pull.add_argument('--decompress', dest="decompress", 
+                      help="extract a gzip compressed image for legacy (prior to 2.4) containers", 
                       default=False, action='store_true')
 
     # List or search labels
diff --git a/singularity/registry/main/pull.py b/singularity/registry/main/pull.py
@@ -35,4 +35,4 @@ def main(args,parser,subparser):
     sreg = Client(secrets=args.secrets)
     response = sreg.pull(images=args.image,
                          file_name=args.name,
-                         decompress=not args.nodecompress)
+                         decompress=args.decompress)
diff --git a/singularity/registry/main/push.py b/singularity/registry/main/push.py
@@ -44,6 +44,4 @@ def main(args,parser,subparser):
     response = sreg.push(path=image,
                          name=args.name,
                          tag=args.tag,
-                         compress=not args.nocompress)
-
-
+                         compress=args.compress)
diff --git a/singularity/tests/test_analysis_classify.py b/singularity/tests/test_analysis_classify.py
diff --git a/singularity/tests/test_hub.py b/singularity/tests/test_hub.py
@@ -58,7 +58,8 @@ def tearDown(self):
     def test_get_collection(self):
         print('Testing client.get_collection command')
         result = self.cli.get_collection(self.collection)
-        for key in ['add_date', 'name', 'enabled', 'modify_date']:
+        for key in ['add_date', 'name', 'metadata', 'modify_date']:
+            print(key)
             self.assertTrue(key in result)
 
     def test_get_container(self):
diff --git a/singularity/tests/test_reproduce.py b/singularity/tests/test_reproduce.py
@@ -146,6 +146,7 @@ def test_extract_guts(self):
                             file_filter=levels['REPLICATE'])
         for key in ['root_owned','sizes','hashes']:
             self.assertTrue(key in guts)
+        tar.close()
 
     def test_get_image_file_hash(self):
         from singularity.analysis.reproduce import get_image_file_hash
diff --git a/singularity/utils.py b/singularity/utils.py
@@ -35,7 +35,6 @@
 
 import shutil
 import json
-import simplejson
 from singularity.logger import bot
 from subprocess import (
     Popen,
@@ -136,14 +135,14 @@ def write_json(json_obj, filename, mode="w", print_pretty=True):
     with open(filename, mode) as filey:
         if print_pretty:
             filey.writelines(
-                simplejson.dumps(
+                json.dumps(
                     json_obj,
                     indent=4,
                     separators=(
                         ',',
                         ': ')))
         else:
-            filey.writelines(simplejson.dumps(json_obj))
+            filey.writelines(json.dumps(json_obj))
     return filename
 
 
diff --git a/singularity/version.py b/singularity/version.py
