Ensure that prioritize_direct_download() retains Seqera Singularity Containers and write additional test.

Author: MatthiasZepper

CHANGELOG.md

@@ -11,6 +11,8 @@
 
 ### Download
 
+- First steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Modify `prioritize_direct_download()` to retain Seqera Singularity https:// Container URIs and hardcode Seqera Containers into `gather_registries()` ([#3244](https://github.com/nf-core/tools/pull/3244)).
+
 ### Linting
 
 ### Modules

nf_core/pipelines/download.py

@@ -970,7 +970,7 @@ def rectify_raw_container_matches(self, raw_findings):
         """
         return self.prioritize_direct_download(cleaned_matches)
 
-    def prioritize_direct_download(self, container_list):
+    def prioritize_direct_download(self, container_list: List[str]) -> List[str]:
         """
         Helper function that takes a list of container images (URLs and Docker URIs),
         eliminates all Docker URIs for which also a URL is contained and returns the
@@ -993,13 +993,31 @@ def prioritize_direct_download(self, container_list):
         we want to keep it and not replace with with whatever we have now (which might be the Docker URI).
 
         A regex that matches http, r"^$|^http" could thus be used to prioritize the Docker URIs over http Downloads
+
+        We also need to handle a special case: The https:// Singularity downloads from Seqera Containers all end in 'data', although
+        they are not equivalent:
+
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/63/6397750e9730a3fbcc5b4c43f14bd141c64c723fd7dad80e47921a68a7c3cd21/data'
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c2/c262fc09eca59edb5a724080eeceb00fb06396f510aefb229c2d2c6897e63975/data'
+
         """
-        d = {}
+        d: dict[str, str] = {}
+        seqera_containers: list[str] = []
+        all_others: list[str] = []
+
         for c in container_list:
+            if bool(re.search(r"/data$", c)):
+                seqera_containers.append(c)
+            else:
+                all_others.append(c)
+
+        for c in all_others:
             if re.match(r"^$|(?!^http)", d.get(k := re.sub(".*/(.*)", "\\1", c), "")):
                 log.debug(f"{c} matches and will be saved as {k}")
                 d[k] = c
-        return sorted(list(d.values()))
+
+        # combine deduplicated others and Seqera containers
+        return sorted(list(d.values()) + seqera_containers)
 
     def gather_registries(self, workflow_directory: str) -> None:
         """Fetch the registries from the pipeline config and CLI arguments and store them in a set.

tests/pipelines/test_download.py

@@ -272,6 +272,60 @@ def test_find_container_images_modules(self, tmp_path, mock_fetch_wf_config):
 
         assert "community.wave.seqera.io/library/coreutils:9.5--ae99c88a9b28c264" in download_obj.containers
 
+    #
+    # Test for 'prioritize_direct_download'
+    #
+    @with_temporary_folder
+    def test_prioritize_direct_download(self, tmp_path):
+        download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path)
+
+        # tests deduplication and https priority as well as Seqera Container exception
+
+        test_container = [
+            "https://depot.galaxyproject.org/singularity/ubuntu:22.04",
+            "nf-core/ubuntu:22.04",
+            "biocontainers/umi-transfer:1.5.0--h715e4b3_0",
+            "https://depot.galaxyproject.org/singularity/umi-transfer:1.5.0--h715e4b3_0",
+            "biocontainers/umi-transfer:1.5.0--h715e4b3_0",
+            "quay.io/nf-core/sortmerna:4.3.7--6502243397c065ba",
+            "nf-core/sortmerna:4.3.7--6502243397c065ba",
+            "https://depot.galaxyproject.org/singularity/sortmerna:4.3.7--hdbdd923_1",
+            "https://depot.galaxyproject.org/singularity/sortmerna:4.3.7--hdbdd923_0",
+            "https://depot.galaxyproject.org/singularity/sortmerna:4.2.0--h9ee0642_1",
+            "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/63/6397750e9730a3fbcc5b4c43f14bd141c64c723fd7dad80e47921a68a7c3cd21/data",
+            "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c2/c262fc09eca59edb5a724080eeceb00fb06396f510aefb229c2d2c6897e63975/data",
+        ]
+
+        result = download_obj.prioritize_direct_download(test_container)
+
+        # Verify that the priority works for regular https downloads (https encountered first)
+        assert "https://depot.galaxyproject.org/singularity/ubuntu:22.04" in result
+        assert "nf-core/ubuntu:22.04" not in result
+
+        # Verify that the priority works for regular https downloads (https encountered second)
+        assert "biocontainers/umi-transfer:1.5.0--h715e4b3_0" not in result
+        assert "https://depot.galaxyproject.org/singularity/umi-transfer:1.5.0--h715e4b3_0" in result
+
+        # Verify that the priority works for images with and without explicit registry
+        # No priority here, though - the first is retained.
+        assert "nf-core/sortmerna:4.3.7--6502243397c065ba" in result
+        assert "quay.io/nf-core/sortmerna:4.3.7--6502243397c065ba" not in result
+
+        # Verify that different versions of the same tool and different build numbers are retained
+        assert "https://depot.galaxyproject.org/singularity/sortmerna:4.3.7--hdbdd923_1" in result
+        assert "https://depot.galaxyproject.org/singularity/sortmerna:4.3.7--hdbdd923_0" in result
+        assert "https://depot.galaxyproject.org/singularity/sortmerna:4.2.0--h9ee0642_1" in result
+
+        # Verify that Seqera containers are not deduplicated
+        assert (
+            "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/63/6397750e9730a3fbcc5b4c43f14bd141c64c723fd7dad80e47921a68a7c3cd21/data"
+            in result
+        )
+        assert (
+            "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c2/c262fc09eca59edb5a724080eeceb00fb06396f510aefb229c2d2c6897e63975/data"
+            in result
+        )
+
     #
     # Tests for 'singularity_pull_image'
     #