Check singularity image already cached

If there is already exists singularity image under cache directory , `singularity pull` causes error.
To avoid error, check whether singularity image is exists or not.

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: manabuishii

cwl_utils/docker_extract.py

@@ -37,6 +37,9 @@ def arg_parser() -> argparse.ArgumentParser:
         help="Specify which command to use to run OCI containers. "
         "Defaults to 'docker' (or 'singularity' if --singularity/-s is passed).",
     )
+    parser.add_argument(
+        "--force-download", help="Force pulling a newer container.", action="store_true"
+    )
     return parser
 
 
@@ -64,6 +67,7 @@ def run(args: argparse.Namespace) -> List[cwl.DockerRequirement]:
                 args.container_engine
                 if args.container_engine is not None
                 else "singularity",
+                args.force_download,
             )
         else:
             image_puller = DockerImagePuller(
@@ -72,6 +76,7 @@ def run(args: argparse.Namespace) -> List[cwl.DockerRequirement]:
                 args.container_engine
                 if args.container_engine is not None
                 else "docker",
+                args.force_download,
             )
         image_puller.save_docker_image()
     return reqs

cwl_utils/image_puller.py

@@ -16,11 +16,14 @@
 
 
 class ImagePuller(ABC):
-    def __init__(self, req: str, save_directory: Union[str, Path], cmd: str) -> None:
+    def __init__(
+        self, req: str, save_directory: Union[str, Path], cmd: str, force_pull: bool
+    ) -> None:
         """Create an ImagePuller."""
         self.req = req
         self.save_directory = save_directory
         self.cmd = cmd
+        self.force_pull = force_pull
 
     @abstractmethod
     def get_image_name(self) -> str:
@@ -62,17 +65,18 @@ def save_docker_image(self) -> None:
         _LOGGER.info(f"Pulling {self.req} with Docker...")
         cmd_pull = [self.cmd, "pull", self.req]
         ImagePuller._run_command_pull(cmd_pull)
+        dest = os.path.join(self.save_directory, self.get_image_name())
+        if self.force_pull:
+            os.remove(dest)
         cmd_save = [
             self.cmd,
             "save",
             "-o",
-            os.path.join(self.save_directory, self.get_image_name()),
+            dest,
             self.req,
         ]
         subprocess.run(cmd_save, check=True)  # nosec
-        _LOGGER.info(
-            f"Image successfully pulled: {self.save_directory}/{self.get_image_name()}"
-        )
+        _LOGGER.info(f"Image successfully pulled: {dest!r}.")
         print(self.generate_udocker_loading_command())
 
 
@@ -98,15 +102,27 @@ def get_image_name(self) -> str:
         return f"{image_name}{suffix}"
 
     def save_docker_image(self) -> None:
-        """Pull down the Docker container image in the Singularity image format."""
+        """Pull down the Docker software container image and save it in the Singularity image format."""
+        if (
+            os.path.exists(os.path.join(self.save_directory, self.get_image_name()))
+            and not self.force_pull
+        ):
+            _LOGGER.info(f"Already cached {self.req} with Singularity.")
+            return
         _LOGGER.info(f"Pulling {self.req} with Singularity...")
         cmd_pull = [
             self.cmd,
             "pull",
-            "--name",
-            os.path.join(self.save_directory, self.get_image_name()),
-            f"docker://{self.req}",
         ]
+        if self.force_pull:
+            cmd_pull.append("--force")
+        cmd_pull.extend(
+            [
+                "--name",
+                os.path.join(self.save_directory, self.get_image_name()),
+                f"docker://{self.req}",
+            ]
+        )
         ImagePuller._run_command_pull(cmd_pull)
         _LOGGER.info(
             f"Image successfully pulled: {self.save_directory}/{self.get_image_name()}"

tests/test_docker_extract.py

@@ -31,6 +31,42 @@ def test_container_extraction(target: str, engine: str, tmp_path: Path) -> None:
     assert len(list(tmp_path.iterdir())) == 1
 
 
+@pytest.mark.parametrize(
+    ("engine"),
+    [
+        pytest.param("docker", marks=needs_docker),
+        pytest.param("podman", marks=needs_podman),
+        pytest.param("singularity", marks=needs_singularity),
+    ],
+)
+def test_container_extraction_force(engine: str, tmp_path: Path) -> None:
+    """Test force pull container extraction."""
+
+    args = [
+        str(tmp_path),
+        get_data("testdata/md5sum.cwl"),
+        "--container-engine",
+        engine,
+    ]
+    if engine == "singularity":
+        args.append("--singularity")
+    reqs = run(arg_parser().parse_args(args))
+    assert len(reqs) == 1
+    assert len(list(tmp_path.iterdir())) == 1
+    args = [
+        str(tmp_path),
+        get_data("testdata/md5sum.cwl"),
+        "--container-engine",
+        engine,
+        "--force-download",
+    ]
+    if engine == "singularity":
+        args.append("--singularity")
+    reqs = run(arg_parser().parse_args(args))
+    assert len(reqs) == 1
+    assert len(list(tmp_path.iterdir())) == 1
+
+
 @pytest.mark.parametrize(
     ("engine"),
     [