feat: handle singularity/apptainer sandbox images (#2166)

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: natthan-pigoux

cwltool/argparser.py

@@ -543,6 +543,16 @@ def arg_parser() -> argparse.ArgumentParser:
         dest="pull_image",
     )
 
+    container_group.add_argument(
+        "--singularity-sandbox-path",
+        default=None,
+        type=str,
+        help="Singularity/Apptainer sandbox image base path. "
+        "Will use a pre-existing sandbox image. "
+        "Will be prepended to the dockerPull path. "
+        "Equivalent to use CWL_SINGULARITY_IMAGES variable. ",
+        dest="image_base_path",
+    )
     container_group.add_argument(
         "--force-docker-pull",
         action="store_true",

cwltool/context.py

@@ -151,6 +151,7 @@ def __init__(self, kwargs: dict[str, Any] | None = None) -> None:
         self.streaming_allowed: bool = False
 
         self.singularity: bool = False
+        self.image_base_path: str | None = None
         self.podman: bool = False
         self.debug: bool = False
         self.compute_checksum: bool = True

cwltool/docker.py

@@ -201,6 +201,7 @@ def get_from_requirements(
         pull_image: bool,
         force_pull: bool,
         tmp_outdir_prefix: str,
+        image_base_path: str | None = None,
     ) -> str | None:
         if not shutil.which(self.docker_exec):
             raise WorkflowException(f"{self.docker_exec} executable is not available")

cwltool/job.py

@@ -626,6 +626,7 @@ def get_from_requirements(
         pull_image: bool,
         force_pull: bool,
         tmp_outdir_prefix: str,
+        image_base_path: str | None = None,
     ) -> str | None:
         pass
 
@@ -787,6 +788,7 @@ def run(
                             runtimeContext.pull_image,
                             runtimeContext.force_docker_pull,
                             runtimeContext.tmp_outdir_prefix,
+                            runtimeContext.image_base_path,
                         )
                     )
                 if img_id is None:

cwltool/singularity.py

@@ -2,6 +2,7 @@
 
 import copy
 import hashlib
+import json
 import logging
 import os
 import os.path
@@ -10,9 +11,10 @@
 import sys
 import threading
 from collections.abc import Callable, MutableMapping
-from subprocess import check_call, check_output  # nosec
+from subprocess import check_call, check_output, run  # nosec
 from typing import cast
 
+from mypy_extensions import mypyc_attr
 from packaging.version import Version
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
@@ -164,6 +166,30 @@ def _normalize_sif_id(string: str) -> str:
     return string.replace("/", "_") + ".sif"
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
+def _inspect_singularity_sandbox_image(path: str) -> bool:
+    """Inspect singularity sandbox image to be sure it is not an empty directory."""
+    cmd = [
+        "singularity",
+        "inspect",
+        "--json",
+        path,
+    ]
+    try:
+        result = run(cmd, capture_output=True, text=True)  # nosec
+    except Exception:
+        return False
+
+    if result.returncode == 0:
+        try:
+            output = json.loads(result.stdout)
+        except json.JSONDecodeError:
+            return False
+        if output.get("data", {}).get("attributes", {}):
+            return True
+    return False
+
+
 class SingularityCommandLineJob(ContainerCommandLineJob):
     def __init__(
         self,
@@ -183,6 +209,7 @@ def get_image(
         pull_image: bool,
         tmp_outdir_prefix: str,
         force_pull: bool = False,
+        sandbox_base_path: str | None = None,
     ) -> bool:
         """
         Acquire the software container image in the specified dockerRequirement.
@@ -201,17 +228,34 @@ def get_image(
 
         with _IMAGES_LOCK:
             if "dockerImageId" in dockerRequirement:
-                if (d_image_id := dockerRequirement["dockerImageId"]) in _IMAGES:
+                d_image_id = dockerRequirement["dockerImageId"]
+                if d_image_id in _IMAGES:
                     if (resolved_image_id := _IMAGES[d_image_id]) != d_image_id:
                         dockerRequirement["dockerImage_id"] = resolved_image_id
                     return True
+                if d_image_id.startswith("/"):
+                    _logger.info(
+                        SourceLine(dockerRequirement, "dockerImageId").makeError(
+                            f"Non-portable: using an absolute file path in a 'dockerImageId': {d_image_id}"
+                        )
+                    )
 
         docker_req = copy.deepcopy(dockerRequirement)  # thread safety
         if "CWL_SINGULARITY_CACHE" in os.environ:
             cache_folder = os.environ["CWL_SINGULARITY_CACHE"]
         elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ:
             cache_folder = os.environ["SINGULARITY_PULLFOLDER"]
 
+        if os.environ.get("CWL_SINGULARITY_IMAGES", None):
+            image_base_path = os.environ["CWL_SINGULARITY_IMAGES"]
+        else:
+            image_base_path = cache_folder if cache_folder else ""
+
+        if not sandbox_base_path:
+            sandbox_base_path = os.path.abspath(image_base_path)
+        else:
+            sandbox_base_path = os.path.abspath(sandbox_base_path)
+
         if "dockerFile" in docker_req:
             if cache_folder is None:  # if environment variables were not set
                 cache_folder = create_tmp_dir(tmp_outdir_prefix)
@@ -261,21 +305,44 @@ def get_image(
                 )
                 found = True
         elif "dockerImageId" not in docker_req and "dockerPull" in docker_req:
-            match = re.search(pattern=r"([a-z]*://)", string=docker_req["dockerPull"])
-            img_name = _normalize_image_id(docker_req["dockerPull"])
-            candidates.append(img_name)
-            if is_version_3_or_newer():
-                sif_name = _normalize_sif_id(docker_req["dockerPull"])
-                candidates.append(sif_name)
-                docker_req["dockerImageId"] = sif_name
+            # looking for local singularity sandbox image and handle it as a local image
+            sandbox_image_path = os.path.join(sandbox_base_path, dockerRequirement["dockerPull"])
+            if os.path.isdir(sandbox_image_path) and _inspect_singularity_sandbox_image(
+                sandbox_image_path
+            ):
+                docker_req["dockerImageId"] = sandbox_image_path
+                _logger.info(
+                    "Using local Singularity sandbox image found in %s",
+                    sandbox_image_path,
+                )
+                found = True
             else:
-                docker_req["dockerImageId"] = img_name
-            if not match:
-                docker_req["dockerPull"] = "docker://" + docker_req["dockerPull"]
+                match = re.search(pattern=r"([a-z]*://)", string=docker_req["dockerPull"])
+                img_name = _normalize_image_id(docker_req["dockerPull"])
+                candidates.append(img_name)
+                if is_version_3_or_newer():
+                    sif_name = _normalize_sif_id(docker_req["dockerPull"])
+                    candidates.append(sif_name)
+                    docker_req["dockerImageId"] = sif_name
+                else:
+                    docker_req["dockerImageId"] = img_name
+                if not match:
+                    docker_req["dockerPull"] = "docker://" + docker_req["dockerPull"]
         elif "dockerImageId" in docker_req:
-            if os.path.isfile(docker_req["dockerImageId"]):
+            sandbox_image_path = os.path.join(sandbox_base_path, dockerRequirement["dockerImageId"])
+            # handling local singularity sandbox image
+            if os.path.isdir(sandbox_image_path) and _inspect_singularity_sandbox_image(
+                sandbox_image_path
+            ):
+                _logger.info(
+                    "Using local Singularity sandbox image found in %s",
+                    sandbox_image_path,
+                )
+                docker_req["dockerImageId"] = sandbox_image_path
                 found = True
             else:
+                if os.path.isfile(docker_req["dockerImageId"]):
+                    found = True
                 candidates.append(docker_req["dockerImageId"])
                 candidates.append(_normalize_image_id(docker_req["dockerImageId"]))
                 if is_version_3_or_newer():
@@ -294,18 +361,19 @@ def get_image(
                             path = os.path.join(dirpath, entry)
                             if os.path.isfile(path):
                                 _logger.info(
-                                    "Using local copy of Singularity image found in %s",
+                                    "Using local copy of Singularity image %s found in %s",
+                                    entry,
                                     dirpath,
                                 )
                                 docker_req["dockerImageId"] = path
                                 found = True
         if (force_pull or not found) and pull_image:
             cmd: list[str] = []
             if "dockerPull" in docker_req:
-                if cache_folder:
+                if image_base_path:
                     env = os.environ.copy()
                     if is_version_2_6():
-                        env["SINGULARITY_PULLFOLDER"] = cache_folder
+                        env["SINGULARITY_PULLFOLDER"] = image_base_path
                         cmd = [
                             "singularity",
                             "pull",
@@ -320,14 +388,14 @@ def get_image(
                             "pull",
                             "--force",
                             "--name",
-                            "{}/{}".format(cache_folder, docker_req["dockerImageId"]),
+                            "{}/{}".format(image_base_path, docker_req["dockerImageId"]),
                             str(docker_req["dockerPull"]),
                         ]
 
                     _logger.info(str(cmd))
                     check_call(cmd, env=env, stdout=sys.stderr)  # nosec
                     docker_req["dockerImageId"] = "{}/{}".format(
-                        cache_folder, docker_req["dockerImageId"]
+                        image_base_path, docker_req["dockerImageId"]
                     )
                     found = True
                 else:
@@ -385,6 +453,7 @@ def get_from_requirements(
         pull_image: bool,
         force_pull: bool,
         tmp_outdir_prefix: str,
+        image_base_path: str | None = None,
     ) -> str | None:
         """
         Return the filename of the Singularity image.
@@ -394,8 +463,14 @@ def get_from_requirements(
         if not bool(shutil.which("singularity")):
             raise WorkflowException("singularity executable is not available")
 
-        if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull):
-            raise WorkflowException("Container image {} not found".format(r["dockerImageId"]))
+        if not self.get_image(
+            cast(dict[str, str], r),
+            pull_image,
+            tmp_outdir_prefix,
+            force_pull,
+            sandbox_base_path=image_base_path,
+        ):
+            raise WorkflowException(f"Container image not found for {r}")
 
         return os.path.abspath(cast(str, r["dockerImageId"]))
 

tests/sing_local_sandbox_img_id_test.cwl

@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerImageId: container_repo/alpine
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo

tests/sing_local_sandbox_test.cwl

@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: container_repo/alpine
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo

tests/test_docker.py

@@ -185,21 +185,21 @@ def test_podman_required_secfile(tmp_path: Path) -> None:
 def test_singularity_required_secfile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     singularity_dir = tmp_path / "singularity"
     singularity_dir.mkdir()
-    monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
-
-    result_code, stdout, stderr = get_main_output(
-        [
-            "--singularity",
-            "--outdir",
-            str(tmp_path / "out"),
-            get_data("tests/secondary-files-required-container.cwl"),
-        ]
-    )
-    assert result_code == 0, stderr
-    assert (
-        json.loads(stdout)["output"]["secondaryFiles"][0]["checksum"]
-        == "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
-    )
+    with monkeypatch.context() as m:
+        m.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                "--outdir",
+                str(tmp_path / "out"),
+                get_data("tests/secondary-files-required-container.cwl"),
+            ]
+        )
+        assert result_code == 0, stderr
+        assert (
+            json.loads(stdout)["output"]["secondaryFiles"][0]["checksum"]
+            == "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
+        )
 
 
 @needs_docker
@@ -247,23 +247,22 @@ def test_singularity_required_missing_secfile(
 ) -> None:
     singularity_dir = tmp_path / "singularity"
     singularity_dir.mkdir()
-    monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
-    result_code, stdout, stderr = get_main_output(
-        [
-            "--singularity",
-            "--outdir",
-            str(tmp_path),
-            get_data("tests/secondary-files-required-missing-container.cwl"),
-        ]
-    )
-    assert result_code == 1, stderr
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "Job error:" in stderr
-    assert "Error collecting output for parameter 'output'" in stderr
-    assert (
-        "tests/secondary-files-required-missing-container.cwl:16:5: Missing required secondary file"
-    )
-    assert "file.ext3" in stderr
+    with monkeypatch.context() as m:
+        m.setenv("CWL_SINGULARITY_CACHE", str(singularity_dir))
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                "--outdir",
+                str(tmp_path),
+                get_data("tests/secondary-files-required-missing-container.cwl"),
+            ]
+        )
+        assert result_code == 1, stderr
+        stderr = re.sub(r"\s\s+", " ", stderr)
+        assert "Job error:" in stderr
+        assert "Error collecting output for parameter 'output'" in stderr
+        assert "tests/secondary-files-required-missing-container.cwl:16:5: Missing required secondary file"
+        assert "file.ext3" in stderr
 
 
 @needs_docker

tests/test_ext.py

@@ -33,12 +33,13 @@ def test_listing_deep() -> None:
 @needs_docker
 def test_cwltool_options(monkeypatch: pytest.MonkeyPatch) -> None:
     """Check setting options via environment variable."""
-    monkeypatch.setenv("CWLTOOL_OPTIONS", "--enable-ext")
-    params = [
-        get_data("tests/wf/listing_deep.cwl"),
-        get_data("tests/listing-job.yml"),
-    ]
-    assert main(params) == 0
+    with monkeypatch.context() as m:
+        m.setenv("CWLTOOL_OPTIONS", "--enable-ext")
+        params = [
+            get_data("tests/wf/listing_deep.cwl"),
+            get_data("tests/listing-job.yml"),
+        ]
+        assert main(params) == 0
 
 
 @needs_docker