Merge pull request #3706 from ErikDanielsson/container-load-scripts

Add container load scripts for Docker and Podman (#3634 follow up)

Author: JulianFlesch

CHANGELOG.md

@@ -32,6 +32,7 @@
   - Add support for downloading docker images into tar archives
   - Change long flag `--parallel-downloads` to `--parallel`. Short flag remains `-d`.
   - Add pipeline to test data to be compatible with `nextflow inspect`
+- Add container load scripts for Docker and Podman (#3634 follow up) ([#3706](https://github.com/nf-core/tools/pull/3706))
 - Replace arm profile with arm64 and emulate_amd64 profiles ([#3689](https://github.com/nf-core/tools/pull/3689))
 - Update test-datasets list subcommand to output plain text urls and paths for easy copying [#3720](https://github.com/nf-core/tools/pull/3720)
 - Remove workflow.trace from nf-test snapshot ([#3721](https://github.com/nf-core/tools/pull/3721))

nf_core/pipelines/download/docker.py

@@ -14,11 +14,11 @@
 
 import nf_core.utils
 from nf_core.pipelines.download.container_fetcher import ContainerFetcher, ContainerProgress
+from nf_core.pipelines.download.utils import copy_container_load_scripts
 
 log = logging.getLogger(__name__)
 stderr = rich.console.Console(
     stderr=True,
-    style="dim",
     highlight=False,
     force_terminal=nf_core.utils.rich_force_colors(),
 )
@@ -308,21 +308,22 @@ def write_docker_load_message(self) -> None:
         """
         Write a message to the user about how to load the downloaded docker images into the offline docker daemon
         """
-        # There is not direct Nextflow support for loading docker images like we do for Singularity
-        # Instead we give the user a `bash` command to load the downloaded docker images into the offline docker daemon
-        # Courtesy of @vmkalbskopf in https://github.com/nextflow-io/nextflow/discussions/4708
-        # TODO: Should we create a bash script instead?
-        docker_load_command = "ls -1 *.tar | xargs --no-run-if-empty -L 1 docker load -i"
-        indent_spaces = 4
+        # Original command courtesy of @vmkalbskopf in https://github.com/nextflow-io/nextflow/discussions/4708
         docker_img_dir = self.get_container_output_dir()
+        podman_load_script, _ = copy_container_load_scripts("podman", docker_img_dir)
+        docker_load_script, _ = copy_container_load_scripts("docker", docker_img_dir)
+        indent_spaces = 4
         stderr.print(
             "\n"
-            + (1 * indent_spaces * " " + f"Downloaded docker images written to [blue not bold]'{docker_img_dir}'[/]. ")
-            + (0 * indent_spaces * " " + "After copying the pipeline and images to the offline machine, run\n\n")
-            + (2 * indent_spaces * " " + f"[blue bold]{docker_load_command}[/]\n\n")
+            + (1 * indent_spaces * " " + f"Downloaded docker images written to [magenta]'{docker_img_dir}'[/].\n")
+            + (1 * indent_spaces * " " + "After copying the pipeline and images to the offline machine, run\n\n")
+            + (
+                2 * indent_spaces * " "
+                + f"[green]./{docker_load_script}[/] (or [green]./{podman_load_script}[/] (experimental))\n\n"
+            )
             + (
                 1 * indent_spaces * " "
-                + f"inside [blue not bold]'{docker_img_dir}'[/] to load the images into the offline docker daemon."
+                + f"inside [magenta]'{docker_img_dir}'[/] to load the images into the offline Docker (Podman) daemon."
             )
             + "\n"
         )

nf_core/pipelines/download/load_scripts/__init__.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail # Ensure that the script exits as early as possible
+
+LOGFILE="podman-load.log"
+
+# Clear log
+> "$LOGFILE"
+
+if ! command -v docker &> /dev/null
+then
+    echo "Error: Docker is not installed. Please install it to continue." >&2
+    exit 1
+fi
+
+if ! docker info &> /dev/null; then
+  echo "Error: Docker daemon is not running." >&2
+  exit 1
+fi
+
+echo "Loading tar archives into docker"
+for tarfile in $(ls -1 *.tar); do
+    if output=$(docker load -i $tarfile); then
+        echo "SUCCESS: $tarfile"
+        echo "SUCCESS: $tarfile"                                                >> "$LOGFILE"
+        echo $output                                                            >> "$LOGFILE"
+        echo "----------------------------------------------------------------" >> "$LOGFILE"
+    else
+        echo "ERROR:   $tarfile"
+        echo "ERROR:   $tarfile"                                                >> "$LOGFILE"
+        echo $output                                                            >> "$LOGFILE"
+        echo "----------------------------------------------------------------" >> "$LOGFILE"
+    fi
+done

nf_core/pipelines/download/load_scripts/docker-load.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail # Ensure that the script exits as early as possible
+
+LOGFILE="podman-load.log"
+
+# Clear log
+> "$LOGFILE"
+
+if ! command -v podman &> /dev/null
+then
+    echo "Error: Podman is not installed. Please install it to continue." >&2
+    exit 1
+fi
+
+if ! podman info &> /dev/null; then
+  echo "Error: No Podman machine is ready. Make sure it's installed and configured." >&2
+  exit 1
+fi
+
+PODMAN_LOAD_SINGLE_IMAGE() {
+    local TARFILE="$1"
+
+    # Look for the full image name in the mainfest.json
+    # inside the image tar archive. It is contained in
+    # this RepoTags field
+    REPO_TAG=$(tar -O -xf "$TARFILE" manifest.json | python -c 'import json, sys; print(json.load(sys.stdin)[0]["RepoTags"][0])')
+
+    if [[ -z "$REPO_TAG" || "$REPO_TAG" == "null" ]]; then
+        echo "Error: Could not find RepoTags in $TARFILE" >&2
+        exit 1
+    fi
+
+    # Load the tar archive into podman -- this will typically
+    # save it as localhost/..., which won't work if we want to
+    # use the images in the nf-core pipeline
+    PODMAN_LOAD_OUTPUT=$(podman load -i "$TARFILE" 2>&1)
+
+    # Extract the tag podman created for the image for later renaming
+    LOCAL_TAG=$(echo "$PODMAN_LOAD_OUTPUT" | sed -n 's/^Loaded image(s): \(.*\)$/\1/p')
+
+    if [[ -z "$LOCAL_TAG" ]]; then
+        echo "Error: Could not parse loaded image name from podman load output" >&2
+        exit 1
+    fi
+
+    # Tag the loaded image with the original remote name
+    podman tag "$LOCAL_TAG" "$REPO_TAG"
+
+    echo "Success, loaded and tagged: $REPO_TAG"
+}
+
+echo "Loading tar archives into podman"
+for tarfile in $(ls -1 *.tar); do
+    if output=$(PODMAN_LOAD_SINGLE_IMAGE $tarfile); then
+        echo "SUCCESS: $tarfile"
+        echo "SUCCESS: $tarfile"                                                >> "$LOGFILE"
+        echo $output                                                            >> "$LOGFILE"
+        echo "----------------------------------------------------------------" >> "$LOGFILE"
+    else
+        echo "ERROR:   $tarfile"
+        echo "ERROR:   $tarfile"                                                >> "$LOGFILE"
+        echo $output                                                            >> "$LOGFILE"
+        echo "----------------------------------------------------------------" >> "$LOGFILE"
+    fi
+done

nf_core/pipelines/download/load_scripts/podman-load.sh

@@ -1,12 +1,26 @@
 import contextlib
+import importlib.resources
 import logging
+import shutil
 import tempfile
 from collections.abc import Generator
 from pathlib import Path
 
 log = logging.getLogger(__name__)
 
 
+def copy_container_load_scripts(container_system: str, dest_dir: Path, make_exec: bool = True) -> tuple[str, Path]:
+    container_load_scripts_subpackage = "nf_core.pipelines.download.load_scripts"
+    script_name = f"{container_system}-load.sh"
+    dest_path = dest_dir / script_name
+    with importlib.resources.open_text(container_load_scripts_subpackage, script_name) as src:
+        with open(dest_path, "w") as dest:
+            shutil.copyfileobj(src, dest)
+    if make_exec:
+        dest_path.chmod(0o775)
+    return script_name, dest_path
+
+
 class DownloadError(RuntimeError):
     """A custom exception that is raised when nf-core pipelines download encounters a problem that we already took into consideration.
     In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead.

nf_core/pipelines/download/utils.py

@@ -0,0 +1,127 @@
+"""Tests for the download subcommand of nf-core tools"""
+
+import unittest
+
+import pytest
+import rich.progress_bar
+import rich.table
+import rich.text
+
+from nf_core.pipelines.download.container_fetcher import ContainerProgress
+
+
+class ContainerProgressTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def use_caplog(self, caplog):
+        self._caplog = caplog
+
+    #
+    # Test for 'utils..add/update_main_task'
+    #
+    def test_download_progress_main_task(self):
+        with ContainerProgress() as progress:
+            # No task initially
+            assert progress.tasks == []
+
+            # Add a task, it should be there
+            task_id = progress.add_main_task(total=42)
+            assert task_id == 0
+            assert len(progress.tasks) == 1
+            assert progress.task_ids[0] == task_id
+            assert progress.tasks[0].total == 42
+
+            # Add another task, there should now be two
+            other_task_id = progress.add_task("Another task", total=28)
+            assert other_task_id == 1
+            assert len(progress.tasks) == 2
+            assert progress.task_ids[1] == other_task_id
+            assert progress.tasks[1].total == 28
+
+            progress.update_main_task(total=35)
+            assert progress.tasks[0].total == 35
+            assert progress.tasks[1].total == 28
+
+    #
+    # Test for 'utils.DownloadProgress.sub_task'
+    #
+    def test_download_progress_sub_task(self):
+        with ContainerProgress() as progress:
+            # No task initially
+            assert progress.tasks == []
+
+            # Add a sub-task, it should be there
+            with progress.sub_task("Sub-task", total=42) as sub_task_id:
+                assert sub_task_id == 0
+                assert len(progress.tasks) == 1
+                assert progress.task_ids[0] == sub_task_id
+                assert progress.tasks[0].total == 42
+
+            # The sub-task should be gone now
+            assert progress.tasks == []
+
+            # Add another sub-task, this time that raises an exception
+            with pytest.raises(ValueError):
+                with progress.sub_task("Sub-task", total=28) as sub_task_id:
+                    assert sub_task_id == 1
+                    assert len(progress.tasks) == 1
+                    assert progress.task_ids[0] == sub_task_id
+                    assert progress.tasks[0].total == 28
+                    raise ValueError("This is a test error")
+
+            # The sub-task should also be gone now
+            assert progress.tasks == []
+
+    #
+    # Test for 'utils.DownloadProgress.get_renderables'
+    #
+    def test_download_progress_renderables(self):
+        # Test the "summary" progress type
+        with ContainerProgress() as progress:
+            assert progress.tasks == []
+            progress.add_task("Task 1", progress_type="summary", total=42, completed=11)
+            assert len(progress.tasks) == 1
+
+            renderable = progress.get_renderable()
+            assert isinstance(renderable, rich.console.Group), type(renderable)
+
+            assert len(renderable.renderables) == 1
+            table = renderable.renderables[0]
+            assert isinstance(table, rich.table.Table)
+
+            assert isinstance(table.columns[0]._cells[0], str)
+            assert table.columns[0]._cells[0] == "[magenta]Task 1"
+
+            assert isinstance(table.columns[1]._cells[0], rich.progress_bar.ProgressBar)
+            assert table.columns[1]._cells[0].completed == 11
+            assert table.columns[1]._cells[0].total == 42
+
+            assert isinstance(table.columns[2]._cells[0], str)
+            assert table.columns[2]._cells[0] == "[progress.percentage] 26%"
+
+            assert isinstance(table.columns[3]._cells[0], str)
+            assert table.columns[3]._cells[0] == "•"
+
+            assert isinstance(table.columns[4]._cells[0], str)
+            assert table.columns[4]._cells[0] == "[green]11/42 tasks completed"
+
+
+class ContainerTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def use_caplog(self, caplog):
+        self._caplog = caplog
+
+    @property
+    def logged_levels(self) -> list[str]:
+        return [record.levelname for record in self._caplog.records]
+
+    @property
+    def logged_messages(self) -> list[str]:
+        return [record.message for record in self._caplog.records]
+
+    def __contains__(self, item: str) -> bool:
+        """Allows to check for log messages easily using the in operator inside a test:
+        assert 'my log message' in self
+        """
+        return any(record.message == item for record in self._caplog.records if self._caplog)
+
+    pass