image-puller: support singularity-ce & apptainer

Author: mr-c

.github/workflows/ci-tests.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install 'tox<4' tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
@@ -100,7 +100,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install 'tox<4' tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
         name: Create local branch for diff-quality for PRs

cwl_utils/image_puller.py

@@ -6,6 +6,10 @@
 from abc import ABC, abstractmethod
 from typing import List
 
+from .singularity import get_version as get_singularity_version
+from .singularity import is_version_2_6 as is_singularity_version_2_6
+from .singularity import is_version_3_or_newer as is_singularity_version_3_or_newer
+
 logging.basicConfig(level=logging.INFO)
 _LOGGER = logging.getLogger(__name__)
 
@@ -79,27 +83,15 @@ class SingularityImagePuller(ImagePuller):
     def __init__(self, req: str, save_directory: str) -> None:
         """Create a Singularity-based software container image downloader."""
         super().__init__(req, save_directory)
-        version = subprocess.check_output(  # nosec
-            ["singularity", "--version"], universal_newlines=True
-        )
-        if version.startswith("singularity version "):
-            version = version[20:]
-        self.version = version
-
-    def _is_version_2_6(self) -> bool:
-        return self.version.startswith("2.6")
-
-    def _is_version_3_or_newer(self) -> bool:
-        return int(self.version[0]) >= 3
 
     def get_image_name(self) -> str:
         """Determine the file name appropriate to the installed version of Singularity."""
         image_name = self.req
         for char in self.CHARS_TO_REPLACE:
             image_name = image_name.replace(char, self.NEW_CHAR)
-        if self._is_version_2_6():
+        if is_singularity_version_2_6():
             suffix = ".img"
-        elif self._is_version_3_or_newer():
+        elif is_singularity_version_3_or_newer():
             suffix = ".sif"
         else:
             raise Exception(

cwl_utils/parser/__init__.py

@@ -11,7 +11,6 @@
 from ..errors import GraphTargetMissingException
 from . import cwl_v1_0, cwl_v1_1, cwl_v1_2
 
-
 LoadingOptions = Union[
     cwl_v1_0.LoadingOptions, cwl_v1_1.LoadingOptions, cwl_v1_2.LoadingOptions
 ]

cwl_utils/parser/cwl_v1_0_utils.py

@@ -3,7 +3,7 @@
 import hashlib
 import logging
 from collections import namedtuple
-from typing import Any, Dict, IO, List, MutableSequence, Optional, Tuple, Union, cast
+from typing import IO, Any, Dict, List, MutableSequence, Optional, Tuple, Union, cast
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException

cwl_utils/parser/cwl_v1_1_utils.py

@@ -3,7 +3,7 @@
 import hashlib
 import logging
 from collections import namedtuple
-from typing import Any, Dict, IO, List, MutableSequence, Optional, Tuple, Union, cast
+from typing import IO, Any, Dict, List, MutableSequence, Optional, Tuple, Union, cast
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException

cwl_utils/parser/cwl_v1_2_utils.py

@@ -3,7 +3,7 @@
 import hashlib
 import logging
 from collections import namedtuple
-from typing import Any, Dict, IO, List, MutableSequence, Optional, Tuple, Union, cast
+from typing import IO, Any, Dict, List, MutableSequence, Optional, Tuple, Union, cast
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException

cwl_utils/parser/utils.py

@@ -9,6 +9,7 @@
 from schema_salad.utils import json_dumps
 
 import cwl_utils
+
 from . import (
     Process,
     Workflow,

cwl_utils/singularity.py

@@ -0,0 +1,95 @@
+"""Support Singularity{,-CE} {2,3}.x or Apptainer 1.x."""
+import re
+from subprocess import check_output  # nosec
+from typing import List, Optional, Tuple
+
+from .loghandler import _logger
+
+# Cached version number of singularity
+# This is a list containing major and minor versions as integer.
+# (The number of minor version digits can vary among different distributions,
+#  therefore we need a list here.)
+_SINGULARITY_VERSION: Optional[List[int]] = None
+# Cached flavor / distribution of singularity
+# Can be singularity, singularity-ce or apptainer
+_SINGULARITY_FLAVOR: str = ""
+
+
+def get_version() -> Tuple[List[int], str]:
+    """
+    Parse the output of 'singularity --version' to determine the flavor and version.
+
+    Both pieces of information will be cached.
+
+    :returns: A tuple containing:
+              - A tuple with major and minor version numbers as integer.
+              - A string with the name of the singularity flavor.
+    """
+    global _SINGULARITY_VERSION  # pylint: disable=global-statement
+    global _SINGULARITY_FLAVOR  # pylint: disable=global-statement
+    if _SINGULARITY_VERSION is None:
+        version_output = check_output(  # nosec
+            ["singularity", "--version"], universal_newlines=True
+        ).strip()
+
+        version_match = re.match(r"(.+) version ([0-9\.]+)", version_output)
+        if version_match is None:
+            raise RuntimeError("Output of 'singularity --version' not recognized.")
+
+        version_string = version_match.group(2)
+        _SINGULARITY_VERSION = [int(i) for i in version_string.split(".")]
+        _SINGULARITY_FLAVOR = version_match.group(1)
+
+        _logger.debug(
+            f"Singularity version: {version_string}" " ({_SINGULARITY_FLAVOR}."
+        )
+    return (_SINGULARITY_VERSION, _SINGULARITY_FLAVOR)
+
+
+def is_apptainer_1_or_newer() -> bool:
+    """
+    Check if apptainer singularity distribution is version 1.0 or higher.
+
+    Apptainer v1.0.0 is compatible with SingularityCE 3.9.5.
+    See: https://github.com/apptainer/apptainer/releases
+    """
+    v = get_version()
+    if v[1] != "apptainer":
+        return False
+    return v[0][0] >= 1
+
+
+def is_version_2_6() -> bool:
+    """
+    Check if this singularity version is exactly version 2.6.
+
+    Also returns False if the flavor is not singularity or singularity-ce.
+    """
+    v = get_version()
+    if v[1] != "singularity" and v[1] != "singularity-ce":
+        return False
+    return v[0][0] == 2 and v[0][1] == 6
+
+
+def is_version_3_or_newer() -> bool:
+    """Check if this version is singularity version 3 or newer or equivalent."""
+    if is_apptainer_1_or_newer():
+        return True  # this is equivalent to singularity-ce > 3.9.5
+    v = get_version()
+    return v[0][0] >= 3
+
+
+def is_version_3_1_or_newer() -> bool:
+    """Check if this version is singularity version 3.1 or newer or equivalent."""
+    if is_apptainer_1_or_newer():
+        return True  # this is equivalent to singularity-ce > 3.9.5
+    v = get_version()
+    return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 1)
+
+
+def is_version_3_4_or_newer() -> bool:
+    """Detect if Singularity v3.4+ is available."""
+    if is_apptainer_1_or_newer():
+        return True  # this is equivalent to singularity-ce > 3.9.5
+    v = get_version()
+    return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 4)

tox.ini

@@ -54,7 +54,7 @@ commands =
   py{37,38,39,310,311}-mypy: make mypy
   py37-mypy: make mypy_3.6
 
-whitelist_externals =
+allowlist_externals =
   py{36,37,38,39,310,311}-lint: flake8
   py{36,37,38,39,310,311}-lint: black
   py{36,37,38,39,310,311}-{mypy,shellcheck,lint,unit}: make
@@ -67,7 +67,7 @@ extras =
   py{36,37,38,39,310,311}-unit: pretty
 
 [testenv:py311-pydocstyle]
-whitelist_externals = make
+allowlist_externals = make
 commands = make diff_pydocstyle_report
 deps =
     pydocstyle
@@ -83,5 +83,5 @@ commands =
 deps =
     twine
     wheel
-    readme_renderer[md]
+    readme_renderer[rst]
 skip_install = true