finishing up v2.4 docker to singularity conversion

Author: vsoch

Dockerfile

@@ -1,14 +1,14 @@
 FROM docker:1.13
 
-RUN apk add --update automake libtool m4 autoconf alpine-sdk linux-headers && \
-    wget -qO- https://github.com/gmkurtzer/singularity/archive/2.1.2.tar.gz | tar zxv && \
-    cd singularity-2.1.2 && ./autogen.sh && ./configure --prefix=/usr/local && make && make install && \
-    cd ../ && rm -rf singularity-2.1.2 && \
+RUN apk add --update automake libtool python m4 autoconf alpine-sdk linux-headers && \
+    wget -qO- https://github.com/singularityware/singularity/archive/2.4.tar.gz | tar zxv && \
+    cd singularity-2.4 && ./autogen.sh && ./configure --prefix=/usr/local && make && make install && \
+    cd ../ && rm -rf singularity-2.4 && \
     apk del automake libtool m4 autoconf alpine-sdk linux-headers
 
 RUN mkdir -p /usr/local/var/singularity/mnt
 
-RUN apk add e2fsprogs bash tar rsync
+RUN apk add e2fsprogs bash tar rsync squashfs-tools
 ADD docker2singularity.sh /docker2singularity.sh
 RUN chmod a+x docker2singularity.sh
 

README.md

@@ -1,11 +1,16 @@
 # `docker2singularity`
 
-Are you developing Docker images and you would like to run them on an HPC cluster supporting [Singularity](http://singularity.lbl.gov)? Are you working on Mac or Windows with no easy access to a Linux machine? `docker2singularity` is the simplest way to generate Singularity images. Note that the upcoming Singularity (2.3) release supports import from Docker without sudo, natively, and this is the recommended approach. More information will be provided in the upcoming documentation for 2.3.
+Are you developing Docker images and you would like to run them on an HPC cluster supporting [Singularity](http://singularity.lbl.gov)? Are you working on Mac or Windows with no easy access to a Linux machine? If the pull, import, and general commands to [work with docker images provided by Singularity](http://singularity.lbl.gov/docs-docker) natively do not fit your needs, `docker2singularity` is an alternative way to generate Singularity images. This particular branch is intended for Singularity 2.4, which builds a squashfs image. If you want a legacy version to build ext3 images, see the following other branches:
+
+ - [v2.3](https://github.com/singularityware/docker2singularity/tree/v2.3): Version 2.3 of Singularity. The image format is ext3.
+ - [v2.4](https://github.com/singularityware/docker2singularity/tree/v2.4): Version 2.4 of Singularity. The default image format is squashfs.
+
+Intermediate versions built on [Docker Hub](https://hub.docker.com/r/singularityware/docker2singularity/tags/). A tag with prefix `v` corresponds to a release of the Singularity software, while the others are in reference to releases of Docker.
 
 ## Requirements
 
  - Docker (native Linux or Docker for Mac or Docker for Windows) - to create the Singularity image.
- - Singularity >= 2.1 - to run the Singularity image (**versions 2.0 and older are not supported!**).
+ - Singularity >= 2.1 - to run the Singularity image (**versions 2.0 and older are not supported!**). Note that if running a 2.4 image using earlier versions, not all (later developed) features may be available.
 
 ## Usage
 
@@ -16,7 +21,7 @@ docker run \
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v D:\host\path\where\to\output\singularity\image:/output \
 --privileged -t --rm \
-singularityware/docker2singularity \
+singularityware/docker2singularity:2.4 \
 ubuntu:14.04
 ```
 
@@ -51,6 +56,7 @@ You need to use the `docker info` command to check your docker version and use i
 
 Currently only the 1.10, 1.11, 1.12, and 1.13  versions are supported. If you are using an older version of Docker you will need to upgrade.
 
+
 ### My cluster/HPC requires Singularity images to include specific mount points
 If you are getting `WARNING: Non existant bind point (directory) in container: '/shared_fs'` or a similar error when running your Singularity image that means that your Singularity images require custom mount points. To make the error go away you can specify the mount points required by your system when creating the Singularity image:
 

docker2singularity.sh

@@ -43,6 +43,7 @@ if [ $# == 0 ] ; then
     echo $USAGE
     exit 1;
 fi
+
 mount_points="/oasis /projects /scratch /local-scratch /work /home1 /corral-repl /corral-tacc /beegfs /share/PI /extra /data /oak"
 while getopts ':hm:' option; do
   case "$option" in
@@ -108,9 +109,7 @@ size=`echo $(($size/1000000+1))`
 # I think it would be Ok by adding 1/3 of the size.
 size=`echo $(($size+$size/2))`
 
-echo "Size: $size MB for the singularity container"
-
-
+echo "Size: $size MB for the singularity container, although not needed for squashfs."
 
 
 ################################################################################
@@ -120,23 +119,20 @@ TMPDIR=$(mktemp -u -d)
 mkdir -p $TMPDIR
 
 creation_date=`echo ${creation_date} | cut -c1-10`
-new_container_name=/tmp/$image_name-$creation_date-$container_id.img
-echo "(1/9) Creating an empty image..."
-singularity create -s $size $new_container_name
-echo "(2/9) Importing filesystem..."
-docker export $container_id | singularity import $new_container_name
-docker inspect $container_id >> $TMPDIR/singularity.json
-singularity copy $new_container_name $TMPDIR/singularity.json /
-
-# Bootstrap the image to set up scripts for environment setup
-echo "(3/9) Bootstrapping..."
-singularity bootstrap $new_container_name
-chmod a+rw -R $TMPDIR
+new_container_name=/tmp/$image_name-$creation_date-$container_id.simg
+build_sandbox="${new_container_name}.build"
+echo "(1/9) Creating a build sandbox..."
+mkdir -p ${build_sandbox}
+echo "(2/9) Exporting filesystem..."
+docker export $container_id >> $build_sandbox.tar
+singularity image.import $build_sandbox < $build_sandbox.tar
+docker inspect $container_id >> $build_sandbox/singularity.json
+
 
 ################################################################################
 ### SINGULARITY RUN SCRIPT #####################################################
 ################################################################################
-echo "(4/9) Adding run script..."
+echo "(3/9) Adding run script..."
 CMD=$(docker inspect --format='{{json .Config.Cmd}}' $image)
 if [[ $CMD != [* ]]; then
     if [[ $CMD != "null" ]]; then
@@ -156,61 +152,58 @@ fi
 # Remove quotes, commas, and braces
 ENTRYPOINT=`echo "${ENTRYPOINT//\"/}" | sed 's/\[//g' | sed 's/\]//g' | sed 's/,/ /g'`
 
-echo '#!/bin/sh' > $TMPDIR/singularity
+echo '#!/bin/sh' > $build_sandbox/singularity
 if [[ $ENTRYPOINT != "null" ]]; then
-    echo $ENTRYPOINT '$@' >> $TMPDIR/singularity;
+    echo $ENTRYPOINT '$@' >> $build_sandbox/singularity;
 else
     if [[ $CMD != "null" ]]; then
-        echo $CMD '$@' >> $TMPDIR/singularity;
+        echo $CMD '$@' >> $build_sandbox/singularity;
     fi
 fi
 
-chmod +x $TMPDIR/singularity
-singularity copy $new_container_name $TMPDIR/singularity /
+chmod +x $build_sandbox/singularity
+
 
 ################################################################################
 ### SINGULARITY ENVIRONMENT ####################################################
 ################################################################################
-echo "(5/9) Setting ENV variables..."
+echo "(4/9) Setting ENV variables..."
 docker run --rm --entrypoint="/usr/bin/env" $image > $TMPDIR/docker_environment
-# don't include HOME and HOSTNAME - they mess with local config
+# do not include HOME and HOSTNAME - they mess with local config
 sed -i '/^HOME/d' $TMPDIR/docker_environment
 sed -i '/^HOSTNAME/d' $TMPDIR/docker_environment
 sed -i 's/^/export /' $TMPDIR/docker_environment
 # add quotes around the variable names
 sed -i 's/=/="/' $TMPDIR/docker_environment
 sed -i 's/$/"/' $TMPDIR/docker_environment
-singularity copy $new_container_name $TMPDIR/docker_environment /
-singularity exec --writable $new_container_name /bin/sh -c "echo '. /docker_environment' >> /environment"
+cp $TMPDIR/docker_environment $build_sandbox/docker_environment
+echo '. /docker_environment' >> $build_sandbox/environment
 rm -rf $TMPDIR
 
+
 ################################################################################
 ### Permissions ################################################################
 ################################################################################
-if [ "${mount_points}" ]; then
-echo "(6/9) Adding mount points..."
-singularity exec --writable --contain $new_container_name /bin/sh -c "mkdir -p ${mount_points}"
-else 
-echo "(6/9) Skipping mount points..."
+if [ "${mount_points}" ] ; then
+    echo "(5/9) Adding mount points..."
+    mkdir -p "${build_sandbox}/${mount_points}"
+else
+    echo "(5/9) Skipping mount points..."
 fi 
 
 # making sure that any user can read and execute everything in the container
-echo "(7/9) Fixing permissions..."
-singularity exec --writable --contain $new_container_name /bin/sh -c "find /* -maxdepth 0 -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+r -R '{}' \;"
-buildname=$(singularity exec --contain $new_container_name /bin/sh -c "head -n 1 /etc/issue")
-echo $buildname
-if [[ $buildname =~ Buildroot|Alpine ]] ; then
-    # we're running on a Builroot container and need to use Busybox's find
-    echo "We're running on BusyBox/Buildroot"
-    singularity exec --writable --contain $new_container_name /bin/sh -c "find / -type f -or -type d -perm -u+x,o-x -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+x '{}' \;"
-else
-    echo "We're not running on BusyBox/Buildroot"
-    singularity exec --writable --contain $new_container_name /bin/sh -c "find / -executable -perm -u+x,o-x -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+x '{}' \;"
-fi
+echo "(6/9) Fixing permissions..."
 
-echo "(8/9) Stopping and removing the container..."
+find ${build_sandbox}/* -maxdepth 0 -not -path '${build_sandbox}/dev*' -not -path '${build_sandbox}/proc*' -not -path '${build_sandbox}/sys*' -exec chmod a+r -R '{}' \;
+find ${build_sandbox}/* -type f -or -type d -perm -u+x,o-x -not -path '${build_sandbox}/dev*' -not -path '${build_sandbox}/proc*' -not -path '${build_sandbox}/sys*' -exec chmod a+x '{}' \;
+
+echo "(7/9) Stopping and removing the container..."
 docker stop $container_id
 docker rm $container_id
 
+# Build a final image from the sandbox
+echo "(8/9) Building..."
+singularity build $new_container_name $build_sandbox
+
 echo "(9/9) Moving the image to the output folder..."
-rsync --info=progress2 /tmp/$image_name-$creation_date-$container_id.img /output/
+rsync --info=progress2 $new_container_name /output/