Merge pull request #98 from vsoch/add/google-build

Update reproducibility metrics for 3.* containers

Author: vsoch

.travis.yml

@@ -1,10 +1,29 @@
-language: python
-
 sudo: true
+go:
+    - "1.11"
 
 os: linux
 
-language: python
+language: go
+
+addons:
+  apt:
+    packages:
+      - flawfinder
+      - squashfs-tools
+      - uuid-dev
+      - libuuid1
+      - libffi-dev
+      - libssl-dev
+      - libssl1.0.0
+      - libarchive-dev
+      - libgpgme11-dev
+      - libseccomp-dev
+  homebrew:
+    packages:
+      - squashfs
+    update: true
+
 
 matrix:
   include:
@@ -13,6 +32,9 @@ matrix:
     - os: linux
       env: PYTHON_VERSION=3
 
+install:
+  - # override
+
 before_install:
   - sudo chmod u+x .travis/*
   - .travis/before_install

.travis/before_install

@@ -15,14 +15,14 @@ sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers
 if [ "${PYTHON_VERSION}" == "2" ];
 then
     wget https://repo.continuum.io/archive/Anaconda2-5.0.0.1-Linux-x86_64.sh
-    sudo bash Anaconda2-5.0.0.1-Linux-x86_64.sh -b -p /opt/anaconda2 && sudo chown -R 0755 /opt/anaconda2
-    PATH=/opt/anaconda3/bin:$PATH
+    sudo bash Anaconda2-5.0.0.1-Linux-x86_64.sh -b -p /opt/conda && sudo chown -R 0755 /opt/conda
+    PATH=/opt/conda/bin:$PATH
     rm Anaconda2-5.0.0.1-Linux-x86_64.sh 
     export PATH
 else
     wget https://repo.continuum.io/archive/Anaconda3-5.0.0.1-Linux-x86_64.sh
-    sudo bash Anaconda3-5.0.0.1-Linux-x86_64.sh -b -p /opt/anaconda3 && sudo chown -R 0755 /opt/anaconda3
-    PATH=/opt/anaconda3/bin:$PATH
+    sudo bash Anaconda3-5.0.0.1-Linux-x86_64.sh -b -p /opt/conda && sudo chown -R 0755 /opt/conda
+    PATH=/opt/conda/bin:$PATH
     rm Anaconda3-5.0.0.1-Linux-x86_64.sh
     export PATH
 fi
@@ -32,7 +32,19 @@ which python
 
 conda install -y pandas scikit-learn pygments
 cd $TRAVIS_BUILD_DIR && sudo python -m pip install -r requirements.txt
-python python setup.py install
+/opt/conda/bin/pip uninstall singularity --yes || echo "Not installed"
+/opt/conda/bin/python setup.py install
 
-# Install Singularity (development)
-cd /tmp && git clone https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+# Install Singularity
+
+SINGULARITY_BASE="${GOPATH}/src/github.com/sylabs/singularity"
+export PATH="${GOPATH}/bin:${PATH}"
+
+mkdir -p "${GOPATH}/src/github.com/sylabs"
+cd "${GOPATH}/src/github.com/sylabs"
+
+git clone -b release-3.2 https://github.com/sylabs/singularity
+cd singularity
+./mconfig -v -p /usr/local
+make -j `nproc 2>/dev/null || echo 1` -C ./builddir all
+sudo make -C ./builddir install

.travis/script

@@ -1,2 +1,2 @@
 #!/bin/bash
-python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'
+/opt/conda/bin/python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'

CHANGELOG.md

@@ -15,6 +15,7 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
 
 
 ## [master](https://github.com/singularityware/singularity-python/tree/master)
+ - updating Singularity python for version 3 singularity (3.0.0)
  - removed all client functionality in favor of using spython (2.5)
  - build on Google will not extract file counts, etc.
 

README.md

@@ -2,10 +2,11 @@
 
 [![Build Status](https://travis-ci.org/singularityware/singularity-python.svg?branch=master)](https://travis-ci.org/singularityware/singularity-python)
 
-Singularity Python is a python module and command line tool to provide helpers for working with <a href="https://singularityware.github.io" target="_blank">Singularity</a> containers, specifically providing functions to visualize, package, and compare containers. 
+Singularity Python is a python module and command line tool to provide helpers for working with <a href="https://www.sylabs.io/guides/3.0/user-guide/" target="_blank">Singularity</a> containers, specifically providing functions to visualize, package, and compare containers. 
 
  - If you are looking for the Singularity Python client to pull, build, and otherwise wrap Singularity functions, then please see the repository [singularity-cli](https://singularityhub.github.io/singularity-cli) for the [spython](https://pypi.org/project/spython/) module.
  - If you are looking for local management and interaction with various storage locations (e.g., to pull and inspect containers in Singularity Hub, Registry, or other cloud resource) then please see the [Global Client](https://singularityhub.github.io/sregistry-cli) for the `sregistry` module.
+ - If you are looking for the Google Cloud (original) build functions, see the [v2.5](https://github.com/singularityhub/singularity-python/tree/v2.5) branch.
 
 ## Install
 You have the option to install only the dependencies that are needed for your functionality of interest.

examples/classify/classify_image/count_files.py

@@ -7,13 +7,11 @@
     extension_counts
 )
 
-container = "ubuntu.simg"
+# singularity pull docker://busybox
+container = "busybox_latest.sif"
 
 # Now we might be interested in counting different things
-readme_count = file_counts(container)
-copyright_count = file_counts(container, patterns=['copyright'])
-authors_count = file_counts(container, patterns=['authors','thanks','credit'])
-todo_count = file_counts(container, patterns=['todo'])
+bin_count = file_counts(container, patterns=['bin'])
 
 # Or getting a complete dict of extensions
 extensions = extension_counts(container)

examples/classify/classify_image/derive_tags.py

@@ -15,9 +15,9 @@
 # Default tags will be returned as software in "bin"
 tags = get_tags(container)
 
-# We can also get the raw list of flies
-file_list = get_container_contents(container)['all']
-
 # We can specify other folders of interest
 folders = ['init','init.d','bin','systemd']
 tags = get_tags(container, search_folders=folders)
+
+# We can also get the raw list of flies
+file_list = get_container_contents(container)['all']

examples/reproducibility/assess_replication.py

@@ -1,34 +1,23 @@
 from glob import glob
 
-from singularity.analysis.reproduce import assess_differences
+from singularity.analysis.reproduce import assess_differences, get_level
 
-image_files=glob('*.img')
+# singularity pull docker://ubuntu:14.04
+# singularity pull docker://ubuntu:12.04
 
-# ASSESS DIFFERENCES #######################################
-# returns dictionary with 
-
-report = assess_differences(image_files[0],image_files[1])
+image_files = glob('ubuntu*.sif')
 
-report.keys()
-# dict_keys(['different', 'missing', 'same'])
 
-# These files are equivalent between the images
-print(len(report['same']))
-5663
+# Choose a level that you want to assess based on
+level_filter = {"RECIPE": get_level('RECIPE')}
 
-# These files are present in both, but different
-print(report['different'])
-['./etc/hosts', 
- './.exec', 
- './environment', 
- './etc/mtab', 
- './etc/resolv.conf', 
- './.run', 
- './.shell', 
- './singularity']
+# ASSESS DIFFERENCES #######################################
 
-# These files are found in the first image, but not the second
-print(report['missing'])
-['./var/lib/apt/lists/.wh.archive.ubuntu.com_ubuntu_dists_xenial-updates_main_i18n_Translation-en', 
- './bin/gunzip']
+# Running for all levels, this will take a few minutes
+report = assess_differences(image_files[0], image_files[1], levels=level_filter)
 
+# {'RECIPE': {'difference': [],
+#  'intersect_different': [],
+#  'same': 7,
+#  'union': 14},
+# 'scores': {'RECIPE': 1.0}}

examples/reproducibility/generate_image_hash.py

@@ -7,7 +7,7 @@
     get_image_file_hash    
 )
 
-image_files = glob("*.simg")
+image_files = glob("*.sif")
 image_path = image_files[0]
 
 ########################################################
@@ -30,7 +30,7 @@
 # We can, then generate an image hash, and by default the level "REPLICATION" will be used:
 
 get_image_hash(image_path)
-#'bf8e242931e25ae9496015868ab2e8cc8d156ffd'
+# '4c252c8fb818e4b854a478a1a0df5991'
 
 # But we can also specify a level that we want:
 get_image_hash(image_path,level="IDENTICAL")
@@ -45,10 +45,10 @@
 # of one container at one level!
 digest = get_content_hashes(image_path)
 digest['hashes']['/usr/bin/chfn']
-# 'ee2b438c278011bdac1a3a927e2d37519a8ed9c7'
+# '4b5ee4db88c3b8bfb0cb7cb3a90a7793'
 
 # We can also get a hash of the entire image file, this is done on the
 # binary file and not contents inside.
 
 file_hash = get_image_file_hash(image_path)
-# e'13775a83962ae60744d691eb7f7fd1e96599e656'
+# 'd5349c37fdc2e6f2dca8793732e1c420'

examples/shub/paper/README.md

@@ -1,4 +1,13 @@
-
 # Singularity Hub Paper
 
 This is the code for the Singularity Hub paper. You should be able to view the comparison tree at [result/index.html](result/index.html).
+Note that the version of Singularity Python used for this is older than currently on master,
+you will need to install:
+
+```bash
+git clone -b v2.5 https://www.github.com/singularityhub/singularity-python.git
+cd singularity-python
+python setup.py install
+```
+
+And the images used were also Singularity version 2.2 and 2.3.