Fix deferred datasets in multi-data parameters.

Author: jmchilton

lib/galaxy/job_execution/datasets.py

@@ -7,15 +7,25 @@
     ABCMeta,
     abstractmethod,
 )
-from typing import Union
+from typing import (
+    List,
+    Union,
+)
 
 from galaxy.model import (
     DatasetCollectionElement,
     DatasetInstance,
     HistoryDatasetCollectionAssociation,
 )
 
-DeferrableObjectsT = Union[DatasetInstance, HistoryDatasetCollectionAssociation, DatasetCollectionElement]
+DeferrableObjectsT = Union[
+    DatasetInstance,
+    HistoryDatasetCollectionAssociation,
+    DatasetCollectionElement,
+    List[DatasetInstance],
+    List[Union[HistoryDatasetCollectionAssociation, DatasetCollectionElement]],
+    List[Union[DatasetInstance, HistoryDatasetCollectionAssociation, DatasetCollectionElement]],
+]
 
 
 def dataset_path_rewrites(dataset_paths):

lib/galaxy/tools/evaluation.py

@@ -13,6 +13,7 @@
     List,
     Optional,
     TYPE_CHECKING,
+    Union,
 )
 
 from packaging.version import Version
@@ -287,6 +288,30 @@ def _materialize_objects(
                 assert isinstance(value, (model.HistoryDatasetAssociation, model.LibraryDatasetDatasetAssociation))
                 undeferred = dataset_materializer.ensure_materialized(value)
                 undeferred_objects[key] = undeferred
+            elif isinstance(value, list):
+                undeferred_list: List[
+                    Union[
+                        model.DatasetInstance, model.HistoryDatasetCollectionAssociation, model.DatasetCollectionElement
+                    ]
+                ] = []
+                for potentially_deferred in value:
+                    if isinstance(potentially_deferred, model.DatasetInstance):
+                        if potentially_deferred.state != model.Dataset.states.DEFERRED:
+                            undeferred_list.append(potentially_deferred)
+                        else:
+                            assert isinstance(
+                                potentially_deferred,
+                                (model.HistoryDatasetAssociation, model.LibraryDatasetDatasetAssociation),
+                            )
+                            undeferred = dataset_materializer.ensure_materialized(potentially_deferred)
+                            undeferred_list.append(undeferred)
+                    elif isinstance(
+                        potentially_deferred,
+                        (model.HistoryDatasetCollectionAssociation, model.DatasetCollectionElement),
+                    ):
+                        undeferred_collection = materialize_collection_input(potentially_deferred, dataset_materializer)
+                        undeferred_list.append(undeferred_collection)
+                undeferred_objects[key] = undeferred_list
             else:
                 undeferred_collection = materialize_collection_input(value, dataset_materializer)
                 undeferred_objects[key] = undeferred_collection
@@ -348,10 +373,6 @@ def _deferred_objects(
         Walk input datasets and collections and find inputs that need to be materialized.
         """
         deferred_objects: Dict[str, DeferrableObjectsT] = {}
-        for key, value in input_datasets.items():
-            if value is not None and value.state == model.Dataset.states.DEFERRED:
-                if self._should_materialize_deferred_input(key, value):
-                    deferred_objects[key] = value
 
         def find_deferred_collections(input, value, context, prefixed_name=None, **kwargs):
             if (
@@ -360,6 +381,28 @@ def find_deferred_collections(input, value, context, prefixed_name=None, **kwarg
             ):
                 deferred_objects[prefixed_name] = value
 
+        def find_deferred_datasets(input, value, context, prefixed_name=None, **kwargs):
+            if isinstance(input, DataToolParameter):
+                if isinstance(value, model.DatasetInstance) and value.state == model.Dataset.states.DEFERRED:
+                    deferred_objects[prefixed_name] = value
+                elif isinstance(value, list):
+                    # handle single list reduction as a collection input
+                    if (
+                        value
+                        and len(value) == 1
+                        and isinstance(
+                            value[0], (model.HistoryDatasetCollectionAssociation, model.DatasetCollectionElement)
+                        )
+                    ):
+                        deferred_objects[prefixed_name] = value
+                        return
+
+                    for v in value:
+                        if self._should_materialize_deferred_input(prefixed_name, v):
+                            deferred_objects[prefixed_name] = value
+                            break
+
+        visit_input_values(self.tool.inputs, incoming, find_deferred_datasets)
         visit_input_values(self.tool.inputs, incoming, find_deferred_collections)
 
         return deferred_objects

lib/galaxy_test/api/test_tool_execute.py

@@ -633,3 +633,15 @@ def test_deferred_with_metadata_options_filter(required_tool: RequiredTool, targ
         "ref_names": "chrM",
     }
     required_tool.execute.with_inputs(inputs).assert_has_single_job.with_single_output.with_contents_stripped("chrM")
+
+
+@requires_tool_id("cat_list")
+def test_deferred_multi_input(required_tool: RequiredTool, target_history: TargetHistory):
+    has_src_dict_bed = target_history.with_deferred_dataset_for_test_file("1.bed", ext="bed")
+    has_src_dict_txt = target_history.with_deferred_dataset_for_test_file("1.txt", ext="txt")
+    inputs = {
+        "input1": [has_src_dict_bed.src_dict, has_src_dict_txt.src_dict],
+    }
+    output = required_tool.execute.with_inputs(inputs).assert_has_single_job.with_single_output
+    output.assert_contains("chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-")
+    output.assert_contains("chr1    4225    19670")