adding a bunch of tests

Author: vsoch

.travis.yml

@@ -17,7 +17,7 @@ install:
   - python setup.py sdist
   - python setup.py install
   - pylint --version
-  - git clone https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+  - git clone -b development https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
 
 script:
   - python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'

singularity/analysis/classify.py

@@ -33,6 +33,7 @@
 import os
 import re
 import requests
+from singularity.cli import Singularity
 from singularity.logman import bot
 from singularity.analysis.compare import (
     compare_packages,
@@ -199,34 +200,7 @@ def get_tags(container=None,image_package=None,sudopw=None,search_folders=None,d
 # COUNTING ########################################################################
 ###################################################################################
 
-
-def get_files(container,S=None,tmpdir=None):
-    '''get_files will return a list of files inside a container, sorted by name
-    :param container: the container to use, either shub:// or docker:// or actual
-    '''
-    files = None
-    if tmpdir == None:
-        tmpdir = tempfile.mkdtemp()
-    tmpfile = "%s/files.txt" %tmpdir
-    container_name = remove_uri(container)
-    command = ' ls -LR >> %s 2>/dev/null' %(tmpfile)
-    if S==None:
-        S = Singularity(sudo=None)
-    result = S.execute(container,command)
-    if os.path.exists(tmpfile):
-        os.system("sed -i '/^$/d' %s" %(tmpfile))
-        os.system('sort %s -or %s' %(tmpfile,tmpfile))
-        files = read_file(tmpfile)
-        if len(files) > 0:
-            files = [x for x in files if x.startswith('.')]
-            files = [x.split(container_name)[1:] for x in files]
-            files = [x for x in files if len(x) > 0]
-            files = [x[0] for x in files]    
-    shutil.rmtree(tmpdir)
-    return files
-
 
-
 def file_counts(container=None,patterns=None,image_package=None,sudopw=None,diff=None):
     '''file counts will return a list of files that match one or more regular expressions.
     if no patterns is defined, a default of readme is used. All patterns and files are made

singularity/analysis/compare.py

@@ -4,6 +4,28 @@
 compare.py: part of singularity package
 functions to compare packages and images
 
+The MIT License (MIT)
+
+Copyright (c) 2016-2017 Vanessa Sochat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 '''
 
 from glob import glob
@@ -86,6 +108,12 @@ def compare_singularity_images(image_paths1,image_paths2=None):
         image_paths2 = image_paths1
         repeat = True
 
+    if not isinstance(image_paths1,list):
+        image_paths1 = [image_paths1]
+
+    if not isinstance(image_paths2,list):
+        image_paths2 = [image_paths2]
+
     dfs = pandas.DataFrame(index=image_paths1,columns=image_paths2)
 
     comparisons_done = []

singularity/analysis/utils.py

@@ -4,6 +4,28 @@
 analysis/utils.py: part of singularity package
 utilities for working with analysis / data
 
+The MIT License (MIT)
+
+Copyright (c) 2016-2017 Vanessa Sochat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 '''
 
 from glob import glob
@@ -38,7 +60,7 @@ def get_packages(family=None):
     package_folders = glob("%s/*" %(package_base))
     package_families = [os.path.basename(x) for x in package_folders]
     if family == None:
-        family = "os"
+        family = "docker-os"
     family = family.lower()
     if family in package_families:
         package_folder = "%s/%s" %(package_base,family)

singularity/build/utils.py

@@ -3,6 +3,28 @@
 '''
 build/utils.py: general building util functions
 
+The MIT License (MIT)
+
+Copyright (c) 2016-2017 Vanessa Sochat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 '''
 
 import os

singularity/cli.py

@@ -201,23 +201,38 @@ def importcmd(self,image_path,input_source):
         return image_path
 
 
-    def pull(self,image_path,pull_folder=None):
+    def pull(self,image_path,pull_folder=None,name_by=None):
         '''pull will pull a singularity hub image
         :param image_path: full path to image
+        :param name_by: can be one of commit or hash, default is by image name
         ''' 
+        if name_by is None:
+            name_by = "name"
+        name_by = name_by.lower()
+
         if pull_folder is not None:
-            os.environ['SINGULARITY_PULL_FOLDER'] = pull_folder
+            os.environ['SINGULARITY_PULLFOLDER'] = pull_folder
 
         if not image_path.startswith('shub://'):
             bot.logger.error("pull is only valid for the shub://uri, %s is invalid.",image_name)
             sys.exit(1)           
 
         if self.debug == True:
-            cmd = ['singularity','--debug','pull',image_path]
+            cmd = ['singularity','--debug','pull']
         else:
-            cmd = ['singularity','pull',image_path]
+            cmd = ['singularity','pull']
+
+        if name_by in ['name','commit','hash']:
+            bot.logger.debug("user specified naming pulled image by %s",name_by)
+            name_by = "--%s" %name_by
+            cmd.append(name_by)
+
+        cmd.append(image_path)
+
         output = self.run_command(cmd)
         self.println(output)        
+        if isinstance(output,bytes):
+            output = output.decode('utf-8')
         return output.split("Container is at:")[-1].strip('\n').strip()
 
 
@@ -239,7 +254,7 @@ def run(self,image_path,args=None,writable=False,contain=False):
 
         if args is not None:        
             if not isinstance(args,list):
-                args = command.split(' ')
+                args = args.split(' ')
             cmd = cmd + args
 
         result = self.run_command(cmd,sudo=sudo)

singularity/reproduce.py

@@ -435,7 +435,7 @@ def get_content_hashes(image_path,level=None,regexp=None,include_files=None,tag_
         file_filter = level_filter
 
     elif level is None:
-        file_filter = get_level("RECIPE",version=version,
+        file_filter = get_level("REPLICATE",version=version,
                                 skip_files=skip_files,
                                 include_files=include_files)
 

singularity/tests/test_analysis_classify.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+
+'''
+Test analysis classification functions
+
+The MIT License (MIT)
+
+Copyright (c) 2016-2017 Vanessa Sochat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+'''
+
+
+from numpy.testing import (
+    assert_array_equal, 
+    assert_almost_equal, 
+    assert_equal
+)
+
+from singularity.cli import get_image
+import unittest
+import tempfile
+import shutil
+import json
+import os
+
+class TestAnalysisClassify(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.container = get_image('docker://ubuntu:16.04')
+        self.comparator = get_image('docker://ubuntu:12.04')
+        
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+
+    def test_get_diff(self):
+        print("Testing singularity.analysis.classify.get_diff")
+        from singularity.analysis.classify import get_diff
+        diff = get_diff(self.container)
+        for key in ['.singularity.d', 'lists', 'actions', 'user', 'env']:
+            self.assertTrue(key in diff)
+
+
+    def test_estimate_os(self):
+        print("Testing singularity.analysis.classify.estimate_os")
+        from singularity.analysis.classify import estimate_os
+        estimated_os = estimate_os(self.container)
+        self.assertEqual(estimated_os,'ubuntu:16.10')
+
+
+    def test_get_tags(self):
+        print("Testing singularity.analysis.classify.get_tags")
+        from singularity.analysis.classify import get_tags
+        tags = get_tags(self.container)
+
+        print("Case 1: Testing that empty OS (with no additions) returns no relevant tags")
+        self.assertEqual(estimated_os,'ubuntu:16.10')
+        self.assertTrue(len(tags)==0)
+
+
+    def test_file_counts(self):
+        print("Testing singularity.analysis.classify.file_counts")
+        from singularity.analysis.classify import file_counts
+        counts = file_counts(self.container)
+
+
+    def test_extension_counts(self):
+        print("Testing singularity.analysis.classify.extension_counts")
+        from singularity.analysis.classify import extension_counts
+        counts = extension_counts(self.container)
+        self.assertTrue(len(counts)>0)
+
+
+
+if __name__ == '__main__':
+    unittest.main()