Merge pull request #20916 from mvdbeek/from_work_dir_directory

Add support for picking ``from_work_dir`` directory

Author: mvdbeek

client/src/api/schema/schema.ts

@@ -14258,6 +14258,11 @@ export interface components {
              * @description Parameter name. Used when referencing parameter in workflows.
              */
             name?: string | null;
+            /**
+             * precreate_directory
+             * @default false
+             */
+            precreate_directory: boolean | null;
             /**
              * @description discriminator enum property added by openapi-typescript
              * @enum {string}

client/src/components/Tool/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(//Users/mvandenb/src/galaxy/lib/galaxy/tool_util_models/**)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

client/src/components/Tool/ToolSourceSchema.json

@@ -147,7 +147,7 @@ proto-plus==1.26.1
 protobuf==6.32.1
 prov==1.5.1
 psutil==7.1.0
-pulsar-galaxy-lib==0.15.10
+pulsar-galaxy-lib==0.15.11
 pyasn1==0.6.1
 pyasn1-modules==0.4.2
 pycparser==2.23 ; (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy') or (implementation_name == 'pypy' and platform_python_implementation == 'PyPy')

lib/galaxy/dependencies/pinned-requirements.txt

@@ -288,12 +288,18 @@ def compute_outputs(self) -> None:
             if da_false_path and not os.path.exists(da_false_path):
                 with open(da_false_path, "ab"):
                     pass
+            real_path = da.dataset.get_file_name(sync_cache=False)
+            false_extra_files_path = os.path.join(
+                os.path.dirname(da_false_path or real_path), da.dataset.dataset.extra_files_path_name
+            )
+
             mutable = da.dataset.dataset.external_filename is None
             dataset_path = DatasetPath(
                 da.dataset.dataset.id,
-                da.dataset.get_file_name(sync_cache=False),
+                real_path,
                 false_path=da_false_path,
                 mutable=mutable,
+                false_extra_files_path=false_extra_files_path,
             )
             job_outputs.append(JobOutput(da.name, da.dataset, dataset_path))
 

lib/galaxy/job_execution/setup.py

@@ -3015,7 +3015,7 @@ def setup_external_metadata(
     def get_output_destination(self, output_path):
         """
         Destination for outputs marked as from_work_dir. These must be copied with
-        the same basenme as the path for the ultimate output destination. This is
+        the same basename as the path for the ultimate output destination. This is
         required in the task case so they can be merged.
         """
         return os.path.join(self.working_directory, os.path.basename(output_path))

lib/galaxy/jobs/__init__.py

@@ -288,13 +288,17 @@ def __handle_metadata(
 
 def __copy_if_exists_command(work_dir_output):
     source_file, destination = work_dir_output
+    is_directory = True if destination.endswith("_files") else False
+    test_flag = "-d" if is_directory else "-f"
+    recursive_flag = " -r" if is_directory else ""
+    delete_destination_dir = f" rmdir {destination}; " if is_directory else ""
     if "?" in source_file or "*" in source_file:
         source_file = source_file.replace("*", '"*"').replace("?", '"?"')
     # Check if source and destination exist.
     # Users can purge outputs before the job completes,
     # in that case we don't want to copy the output to a purged path.
     # Static, non work_dir_output files are handled in job_finish code.
-    return f'\nif [ -f "{source_file}" -a -f "{destination}" ] ; then cp "{source_file}" "{destination}" ; fi'
+    return f'\nif [ {test_flag} "{source_file}" -a {test_flag} "{destination}" ] ; then{delete_destination_dir} cp{recursive_flag} "{source_file}" "{destination}" ; fi'
 
 
 class CommandsBuilder:

lib/galaxy/jobs/command_factory.py

@@ -373,11 +373,13 @@ def get_work_dir_outputs(
         # Set up dict of dataset id --> output path; output path can be real or
         # false depending on outputs_to_working_directory
         output_paths = {}
+        output_extra_paths = {}
         for dataset_path in job_wrapper.job_io.get_output_fnames():
             path = dataset_path.real_path
             if asbool(job_wrapper.get_destination_configuration("outputs_to_working_directory", False)):
                 path = dataset_path.false_path
             output_paths[dataset_path.dataset_id] = path
+            output_extra_paths[dataset_path.dataset_id] = dataset_path.false_extra_files_path
 
         output_pairs = []
         # Walk job's output associations to find and use from_work_dir attributes.
@@ -397,9 +399,15 @@ def get_work_dir_outputs(
                     # Copy from working dir to HDA.
                     # TODO: move instead of copy to save time?
                     source_file = os.path.join(tool_working_directory, hda_tool_output.from_work_dir)
-                    destination = job_wrapper.get_output_destination(output_paths[dataset.dataset_id])
+                    if hda_tool_output.precreate_directory:
+                        # precreate directory, allows using `-d` check to avoid copying data to purged outputs
+                        dataset.dataset.create_extra_files_path()
+                        output_path = output_extra_paths[dataset.dataset_id]
+                        os.makedirs(output_path, exist_ok=True)
+                    else:
+                        output_path = output_paths[dataset.dataset_id]
                     if in_directory(source_file, tool_working_directory):
-                        output_pairs.append((source_file, destination))
+                        output_pairs.append((source_file, job_wrapper.get_output_destination(output_path)))
                     else:
                         # Security violation.
                         log.exception(

lib/galaxy/jobs/runners/__init__.py

@@ -52,6 +52,7 @@ class ToolOutputBase(Dictifiable):
     name: str
     label: Optional[str]
     hidden: bool
+    precreate_directory: bool
 
     def __init__(
         self,
@@ -70,6 +71,7 @@ def __init__(
         self.hidden = hidden
         self.collection = False
         self.from_expression = from_expression
+        self.precreate_directory = False
 
     def to_dict(self, view="collection", value_mapper=None, app=None):
         return super().to_dict(view=view, value_mapper=value_mapper)
@@ -131,6 +133,7 @@ def __init__(
         self.change_format: List[ChangeFormatModel] = []
         self.implicit = implicit
         self.from_work_dir: Optional[str] = None
+        self.precreate_directory: bool = False
         self.dataset_collector_descriptions: List[DatasetCollectionDescription] = []
         self.default_identifier_source: Optional[str] = None
         self.count: Optional[int] = None
@@ -178,6 +181,7 @@ def to_model(self) -> ToolOutputDataModel:
             metadata_source=self.metadata_source,
             discover_datasets=[d.to_model() for d in self.dataset_collector_descriptions],
             from_work_dir=self.from_work_dir,
+            precreate_directory=self.precreate_directory,
         )
 
     @staticmethod
@@ -193,6 +197,7 @@ def from_dict(name: str, output_dict: Dict[str, Any], app: Optional[ToolOutputAc
         output.count = output_dict.get("count", 1)
         output.filters = []
         output.from_work_dir = output_dict.get("from_work_dir")
+        output.precreate_directory = output_dict.get("precreate_directory") or False
         output.hidden = output_dict.get("hidden") or False
         # TODO: implement tool output action group fixes
         if app is not None:
@@ -223,6 +228,7 @@ def __init__(self, name, output_type, from_expression, label=None, filters=None,
         self.change_format = []
         self.implicit = False
         self.from_work_dir = None
+        self.precreate_directory = False
 
         self.dataset_collector_descriptions = []
 

lib/galaxy/tool_util/parser/output_objects.py

@@ -589,6 +589,7 @@ def _parse_output(
         output.count = int(data_elem.get("count", 1))
         output.filters = data_elem.findall("filter")
         output.from_work_dir = data_elem.get("from_work_dir", None)
+        output.precreate_directory = data_elem.get("precreate_directory") or False
         profile_version = Version(self.parse_profile())
         if output.from_work_dir and profile_version < Version("21.09"):
             # We started quoting from_work_dir outputs in 21.09.