Merge pull request #4 from khillion/master

Allow to pull images with Singularity

Author: mr-c

.travis.yml

@@ -13,4 +13,5 @@ services:
 install:
  - true
 script:
-- docker build . -f .travis/${TARGET}.docker
+- docker build --tag ${TARGET} . -f .travis/${TARGET}.docker
+- docker run -v /var/run/docker.sock:/var/run/docker.sock ${TARGET} /bin/bash -c "$(cat .travis/${TARGET}.run)"

.travis/.singularity_key.txt

@@ -0,0 +1,70 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGiBEQ7TOgRBADvaRsIZ3VZ6Qy7PlDpdMm97m0OfvouOj/HhjOM4M3ECbGn4cYh
+vN1gK586s3sUsUcNQ8LuWvNsYhxYsVTZymCReJMEDxod0U6/z/oIbpWv5svF3kpl
+ogA66Ju/6cZx62RiCSOkskI6A3Waj6xHyEo8AGOPfzbMoOOQ1TS1u9s2FwCgxziL
+wADvKYlDZnWM03QtqIJVD8UEAOks9Q2OqFoqKarj6xTRdOYIBVEp2jhozZUZmLmz
+pKL9E4NKGfixqxdVimFcRUGM5h7R2w7ORqXjCzpiPmgdv3jJLWDnmHLmMYRYQc8p
+5nqo8mxuO3zJugxBemWoacBDd1MJaH7nK20Hsk9L/jvU/qLxPJotMStTnwO+EpsK
+HlihA/9ZpvzR1QWNUd9nSuNR3byJhaXvxqQltsM7tLqAT4qAOJIcMjxr+qESdEbx
+NHM5M1Y21ZynrsQw+Fb1WHXNbP79vzOxHoZR0+OXe8uUpkri2d9iOocre3NUdpOO
+JHtl6cGGTFILt8tSuOVxMT/+nlo038JQB2jARe4B85O0tkPIPbQybmV1cm8uZGVi
+aWFuLm5ldCBhcmNoaXZlIDxtaWNoYWVsLmhhbmtlQGdtYWlsLmNvbT6IRgQQEQgA
+BgUCTVHJKwAKCRCNEUVjdcAkyOvzAJ0abJz+f2a6VZG1c9T8NHMTYh1atwCgt0EE
+3ZZd/2in64jSzu0miqhXbOKISgQQEQIACgUCSotRlwMFAXgACgkQ93+NsjFEvg8n
+JgCfWcdJbILBtpLZCocvOzlLPqJ0Fn0AoI4EpJRxoUnrtzBGUC1MqecU7WsDiGAE
+ExECACAFAkqLUWcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCl0y8BJkml
+qVklAJ4h2V6MdQkSAThF5c2Gkq6eSoIQYQCeM0DWyB9Bl+tTPSTYXwwZi2uoif20
+QmFwc3kuZ3NlLnVuaS1tYWdkZWJ1cmcuZGUgRGViaWFuIEFyY2hpdmUgPG1pY2hh
+ZWwuaGFua2VAZ21haWwuY29tPohGBBARAgAGBQJEO03FAAoJEPd/jbIxRL4PU18A
+n3tn7i4qdlMi8kHbYWFoabsKc9beAJ9sl/leZNCYNMGhz+u6BQgyeLKw94heBBMR
+AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA
+n27DvtZizNEbhz3wRUPQMiQjtqdvAJ9rS9YdPe5h5o5gHx3mw3BSkOttdYheBBMR
+AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA
+oLhwWL+E+2I9lrUf4Lf26quOK9vLAKC9ZpIF2tUirFFkBWnQvu13/TA0SokCHAQQ
+AQIABgUCTSNBgQAKCRDAc9Iof/uem4NpEACQ8jxmaCaS/qk/Y4GiwLA5bvKosG3B
+iARZ2v5UWqCZQ1tS56yKse/lCIzXQqU9BnYW6wOI2rvFf9meLfd8h96peG6oKscs
+fbclLDIf68bBvGBQaD0VYFi/Fk/rxmTQBOCQ3AJZs8O5rIM4gPGE0QGvSZ1h7VRw
+3Uyeg4jKXLIeJn2xEmOJgt3auAR2FyKbzHaX9JCoByJZ/eU23akNl9hgt7ePlpXo
+74KNYC58auuMUhCq3BQDB+II4ERYMcmFp1N5ZG05Cl6jcaRRHDXz+Ax6DWprRI1+
+RH/Yyae6LmKpeJNwd+vM14aawnNO9h8IAQ+aJ3oYZdRhGyybbin3giJ10hmWveg/
+Pey91Nh9vBCHdDkdPU0s9zE7z/PHT0c5ccZRukxfZfkrlWQ5iqu3V064ku5f4PBy
+8UPSkETcjYgDnrdnwqIAO+oVg/SFlfsOzftnwUrvwIcZlXAgtP6MEEAs/38e/JIN
+g4VrpdAy7HMGEUsh6Ah6lvGQr+zBnG44XwKfl7e0uCYkrAzUJRGM5vx9iXvFMcMu
+jv9EBNNBOU8/Y6MBDzGZhgaoeI27nrUvaveJXjAiDKAQWBLjtQjINZ8I9uaSGOul
+8kpbFavE4eS3+KhISrSHe4DuAa3dk9zI+FiPvXY1ZyfQBtNpR+gYFY6VxMbHhY1U
+lSLHO2eUIQLdYbRITmV1cm9EZWJpYW4gQXJjaGl2ZSBLZXkgPHBrZy1leHBwc3kt
+bWFpbnRhaW5lcnNAbGlzdHMuYWxpb3RoLmRlYmlhbi5vcmc+iEYEEBEIAAYFAk1R
+yQYACgkQjRFFY3XAJMgEWwCggx4Gqlcrt76TSMlbU94cESo55AEAoJ3asQEMpe8t
+QUX+5aikw3z1AUoCiEoEEBECAAoFAkqf/3cDBQF4AAoJEPd/jbIxRL4PxyMAoKUI
+RPWlHCj/+HSFfwhos68wcSwmAKChuC00qutDro+AOo+uuq6YoHXj+ohgBBMRAgAg
+BQJKn/8bAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQpdMvASZJpalDggCe
+KF9KOgOPdQbFnKXl8KtHory4EEwAnA7jxgorE6kk2QHEXFSF8LzOOH4GiGMEExEC
+ACMCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAUCSp//RgIZAQAKCRCl0y8BJkml
+qekFAKCRyt4+FoCzmBbRUUP3Cr8PzH++IgCgkno4vdjsWdyAey8e0KpITTXMFrmJ
+AhwEEAECAAYFAk0jQYEACgkQwHPSKH/7npsFfw/+P8B8hpM3+T1fgboBa4R32deu
+n8m6b8vZMXwuo/awQtMpzjem8JGXSUQm8iiX4hDtjq6ZoPrlN8T4jNmviBt/F5jI
+Jji/PYmhq+Zn9s++mfx+aF4IJrcHJWFkg/6kJzn4oSdl/YlvKf4VRCcQNtj4xV87
+GsdamnzU17XapLVMbSaVKh+6Af7ZLDerEH+iAq733HsYaTK+1xKmN7EFVXgS7bZ1
+9C4LTzc97bVHSywpT9yIrg9QQs/1kshfVIHDKyhjF6IwzSVbeGAIL3Oqo5zOMkWv
+7JlEIkkhTyl+FETxNMTMYjAk+Uei3kRodneq3YBF2uFYSEzrXQgHAyn37geiaMYj
+h8wu6a85nG1NS0SdxiZDIePmbvD9vWxFZUWYJ/h9ifsLivWcVXlvHoQ0emd+n2ai
+FhAck2xsuyHgnGIZMHww5IkQdu/TMqvbcR6d8Xulh+C4Tq7ppy+oTLADSBKII++p
+JQioYydRD529EUJgVlhyH27X6YAk3FuRD3zYZRYS2QECiKXvS665o3JRJ0ZSqNgv
+YOom8M0zz6bI9grnUoivMI4o7ISpE4ZwffEd37HVzmraaUHDXRhkulFSf1ImtXoj
+V9nNSM5p/+9eP7OioTZhSote6Vj6Ja1SZeRkXZK7BwqPbdO0VsYOb7G//ZiOlqs+
+paRr92G/pwBfj5Dq8EK5Ag0ERDtM9RAIAN0EJqBPvLN0tEin/y4Fe0R4n+E+zNXg
+bBsq4WidwyUFy3h/6u86FYvegXwUqVS2OsEs5MwPcCVJOfaEthF7I89QJnP9Nfx7
+V5I9yFB53o9ii38BN7X+9gSjpfwXOvf/wIDfggxX8/wRFel37GRB7TiiABRArBez
+s5x+zTXvT++WPhElySj0uY8bjVR6tso+d65K0UesvAa7PPWeRS+3nhqABSFLuTTT
+MMbnVXCGesBrYHlFVXClAYrSIOX8Ub/UnuEYs9+hIV7U4jKzRF9WJhIC1cXHPmOh
+vleAf/I9h/0KahD7HLYud40pNBo5tW8jSfp2/Q8TIE0xxshd51/xy4MAAwUH+wWn
+zsYVk981OKUEXul8JPyPxbw05fOd6gF4MJ3YodO+6dfoyIl3bewk+11KXZQALKaO
+1xmkAEO1RqizPeetoadBVkQBp5xPudsVElUTOX0pTYhkUd3iBilsCYKK1/KQ9KzD
+I+O/lRsm6L9lc6rV0IgPU00P4BAwR+x8Rw7TJFbuS0miR3lP1NSguz+/kpjxzmGP
+LyHJ+LVDYFkk6t0jPXhqFdUY6McUTBDEvavTGlVO062l9APTmmSMVFDsPN/rBes2
+rYhuuT+lDp+gcaS1UoaYCIm9kKOteQBnowX9V74Z+HKEYLtwILaSnNe6/fNSTvyj
+g0z+R+sPCY4nHewbVC+ISQQYEQIACQUCRDtM9QIbDAAKCRCl0y8BJkmlqbecAJ9B
+UdSKVg9H+fQNyP5sbOjj4RDtdACfXHrRHa2+XjJP0dhpvJ8IfvYnQsU=
+=fAJZ
+-----END PGP PUBLIC KEY BLOCK-----

.travis/mypy.run

@@ -1,6 +1,8 @@
 FROM kernsuite/base:5
-RUN docker-apt-install python3-pip
+ADD .travis/.singularity_key.txt /
+RUN docker-apt-install wget
+RUN wget -O/etc/apt/sources.list.d/neurodebian.sources.list http://neuro.debian.net/lists/bionic.us-nh.full && apt-key add /.singularity_key.txt
+RUN docker-apt-install python3-pip singularity-container docker.io
 ADD . /code
 WORKDIR /code
 RUN pip3 install .
-RUN python3 setup.py test

.travis/pep8.run

@@ -0,0 +1 @@
+service docker start && cd /code && python3 setup.py test

.travis/py3.docker

@@ -1,20 +1,41 @@
+# cwl-utils
+
 A collection of scripts to demonstrate the use of the [new Python classes for loading and parsing CWL v1.0 documents](https://github.com/common-workflow-language/cwl-utils/blob/master/cwl_utils/parser_v1_0.py).
 
 
 `cwl_utils/parser_v1_0.py` was created via
 `schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/master/v1.0/CommonWorkflowLanguage.yml`
 
+## Install
+
 Requires Python 3.6.x or Python 3.7
 
 ``` bash
 git clone https://github.com/common-workflow-language/cwl-utils.git
 virtualenv -p python3.6 venv3.6
 source venv3.6/bin/activate
 pip install cwl-utils
-python docker-extract.py path_to_my_workflow.cwl
 ```
 
-to regenerate install `schema_salad` package and run:
+## Usage
+
+### Pull the image with Docker
+
+This is the default behaviour:
+
+```bash
+python docker_extract.py DIRECTORY path_to_my_workflow.cwl
+```
+
+### Pull the image with Singularity
+
+```bash
+python docker_extract.py --singularity DIRECTORY path_to_my_workflow.cwl
+```
+
+## Regenerate
+
+To regenerate install `schema_salad` package and run:
 
 ```
 schema-salad-tool --codegen python \

.travis/py3.run

@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
-import sys
+import argparse
 import os
+import sys
+
+from cwl_utils.image_puller import DockerImagePuller, SingularityImagePuller
 import cwl_utils.parser_v1_0 as cwl
-import subprocess
-import argparse
 
 
 def parse_args():
     parser = argparse.ArgumentParser(
-        description='Tool to save docker images from a cwl workflow \n '
-                    'and generate udocker loading commands.')
+        description='Tool to save docker images from a cwl workflow.')
     parser.add_argument('dir', help='Directory in which to save images')
     parser.add_argument('input', help='Input CWL workflow')
+    parser.add_argument('-s', '--singularity', help='Use singularity to pull the image', action='store_true')
     return parser.parse_args()
 
 
@@ -22,32 +23,11 @@ def main():
     top = cwl.load_document(args.input)
 
     for req in set(traverse(top)):
-        image_name = get_image_name(req)
-        save_docker_image(req, image_name, args.dir)
-
-        print(load_docker_image(image_name))
-
-
-def get_image_name(req):
-    return ''.join(req.split('/')) + '.tar'
-
-
-def load_docker_image(image_name):
-    return f'udocker load -i  {image_name}'
-
-
-def save_docker_image(req, image_name, image_dir):
-    cmd_pull = ['docker', 'pull', req]
-    try:
-        subprocess.run(cmd_pull, check=True, stdout=subprocess.PIPE,
-                       stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as err:
-        if err.output:
-            raise subprocess.SubprocessError(err.output)
-        raise err
-    cmd_save = ['docker', 'save', '-o', os.path.join(image_dir, image_name),
-                req]
-    subprocess.run(cmd_save, check=True)
+        if args.singularity:
+            image_puller = SingularityImagePuller(req, args.dir)
+        else:
+            image_puller = DockerImagePuller(req, args.dir)
+        image_puller.save_docker_image()
 
 
 def extract_docker_requirements(process: cwl.Process):

README.md

@@ -0,0 +1,96 @@
+from abc import ABC, abstractmethod
+import logging
+import os
+import subprocess
+
+logging.basicConfig(level=logging.INFO)
+_LOGGER = logging.getLogger(__name__)
+
+
+class ImagePuller(ABC):
+
+    def __init__(self, req, save_directory):
+        self.req = req
+        self.save_directory = save_directory
+
+    @abstractmethod
+    def get_image_name(self):
+        pass
+
+    @abstractmethod
+    def save_docker_image(self):
+        pass
+
+    @staticmethod
+    def _run_command_pull(cmd_pull):
+        try:
+            subprocess.run(cmd_pull, check=True, stdout=subprocess.PIPE,
+                           stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as err:
+            if err.output:
+                raise subprocess.SubprocessError(err.output)
+            raise err
+
+
+class DockerImagePuller(ImagePuller):
+    """
+    Pull docker image with Docker
+    """
+
+    def get_image_name(self):
+        return ''.join(self.req.split('/')) + '.tar'
+
+    def generate_udocker_loading_command(self):
+        return f'udocker load -i {self.get_image_name()}'
+
+    def save_docker_image(self):
+        _LOGGER.info(f"Pulling {self.req} with Docker...")
+        cmd_pull = ['docker', 'pull', self.req]
+        ImagePuller._run_command_pull(cmd_pull)
+        cmd_save = ['docker', 'save', '-o', os.path.join(self.save_directory,
+                                                         self.get_image_name()),
+                    self.req]
+        subprocess.run(cmd_save, check=True)
+        _LOGGER.info(f"Image successfully pulled: {self.save_directory}/{self.get_image_name()}")
+        print(self.generate_udocker_loading_command())
+
+
+class SingularityImagePuller(ImagePuller):
+    """
+    Pull docker image with Singularity
+    """
+    CHARS_TO_REPLACE = ['/', ':']
+    NEW_CHAR = '-'
+
+    def __init__(self, req, save_directory):
+        super(SingularityImagePuller, self).__init__(req, save_directory)
+        version = subprocess.check_output(["singularity", "--version"], universal_newlines=True)
+        if version.startswith("singularity version "):
+            version = version[20:]
+        self.version = version
+
+    def _is_version_2_6(self):  # type: ()->bool
+        return self.version.startswith("2.6")
+
+    def _is_version_3_or_newer(self):  # type: ()->bool
+        return int(self.version[0]) >= 3
+
+    def get_image_name(self):
+        image_name = self.req
+        for char in self.CHARS_TO_REPLACE:
+            image_name = image_name.replace(char, self.NEW_CHAR)
+        if self._is_version_2_6():
+            suffix = ".img"
+        elif self._is_version_3_or_newer():
+            suffix = ".sif"
+        else:
+            raise Exception("Don't know how to handle this version of singularity: {}.".format(self.version))
+        return f'{image_name}.{suffix}'
+
+    def save_docker_image(self):
+        _LOGGER.info(f"Pulling {self.req} with Singularity...")
+        cmd_pull = ['singularity', 'pull', '--name', os.path.join(self.save_directory,
+                                                        self.get_image_name()),
+                    f'docker://{self.req}']
+        ImagePuller._run_command_pull(cmd_pull)
+        _LOGGER.info(f"Image successfully pulled: {self.save_directory}/{self.get_image_name()}")

cwl_utils/docker_extract.py

@@ -18,4 +18,5 @@
     setup_requires=['pytest-runner'],
     tests_require=['pytest'],
     test_suite='tests',
+    scripts=['cwl_utils/docker_extract.py'],
 )