Do not transfer of output files if destination is given (#38)

Author: JoschD

CHANGELOG.md

@@ -1,5 +1,9 @@
 # `pylhc-submitter` Changelog
 
+## Version 2.0.3
+
+- Fixing `job_submitter`: Do not transfer any output files, when the `output_destination` is given.
+
 ## Version 2.0.2
 
 - Fixing `job_submitter`: Discovers more invalid URIs.

pylhc_submitter/__init__.py

@@ -10,7 +10,7 @@
 __title__ = "pylhc_submitter"
 __description__ = "pylhc-submitter contains scripts to simplify the creation and submission of jobs to HTCondor at CERN"
 __url__ = "https://github.com/pylhc/submitter"
-__version__ = "2.0.2"
+__version__ = "2.0.3"
 __author__ = "pylhc"
 __author_email__ = "pylhc@github.com"
 __license__ = "MIT"

pylhc_submitter/constants/htcondor.py

@@ -21,4 +21,4 @@
     "nextweek",  # 1 w
 )
 
-NOTIFICATIONS = ("always", "complete", "error", "never")
+NOTIFICATIONS = ("always", "complete", "error", "never")

pylhc_submitter/job_submitter.py

@@ -394,7 +394,7 @@ def check_opts(opt):
 
     creation = CreationOpts(**{f.name: opt[f.name] for f in fields(CreationOpts)})
     runner = RunnerOpts(**{f.name: opt[f.name] for f in fields(RunnerOpts)})
-    runner.output_dir = None if opt.output_destination else opt.output_dir
+    runner.output_dir = '""' if opt.output_destination else opt.output_dir  # empty string stops htc transfer of files
     return creation, runner
 
 

pylhc_submitter/submitter/htc_utils.py

@@ -107,7 +107,7 @@ def create_multijob_for_bashfiles(job_df: DataFrame, **kwargs) -> str:
 
     Keyword Args:
         output_dir (str): output directory that will be transferred. Defaults to ``None``.
-        duration (str): max duration of the job. Needs to be one of the ``HTCondor`` Jobflavours.
+        jobflavour (str): max duration of the job. Needs to be one of the ``HTCondor`` Jobflavours.
             Defaults to ``workday``.
         group (str): force use of accounting group. Defaults to ``None``.
         retries (int): maximum amount of retries. Default to ``3``.
@@ -257,8 +257,8 @@ def map_kwargs(add_dict: Dict[str, Any]) -> Dict[str, Any]:
 
     # Predefined mappings 
     htc_map = { # name: mapped_name, choices, default
-        "duration": ("+JobFlavour", JOBFLAVOURS, "workday"),
-        "output_dir": ("transfer_output_files", None, None),
+        "jobflavour": ("+JobFlavour", JOBFLAVOURS, "workday"),
+        "output_dir": ("transfer_output_files", None, '""'),
         "accounting_group": ("+AccountingGroup", None, None),
         "max_retries": ("max_retries", None, 3),
         "notification": ("notification", NOTIFICATIONS, "error"),

pylhc_submitter/submitter/runners.py

@@ -7,7 +7,7 @@
 import logging
 import multiprocessing
 import subprocess
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Dict, Optional, Tuple
 
@@ -31,7 +31,7 @@ class RunnerOpts:
     output_dir: Optional[str] = None  # Name of the output directory, where jobs store data
     ssh: Optional[str] = None         # SSH command
     dryrun: Optional[bool] = False    # Perform only a dry-run, i.e. do all but submit to HTC
-    htc_arguments: Optional[Dict[str, Any]] = None  # Arguments to pass on to htc as keywords
+    htc_arguments: Optional[Dict[str, Any]] = field(default_factory=dict)  # Arguments to pass on to htc as keywords
     run_local: Optional[bool] = False # Run jobs locally
     num_processes: Optional[int] = 4  # Number of processes to run in parallel (locally)
 
@@ -83,7 +83,7 @@ def run_htc(job_df: tfs.TfsDataFrame, opt: RunnerOpts) -> None:
     subfile = htc_utils.make_subfile(
         opt.working_directory, job_df, 
         output_dir=opt.output_dir, 
-        duration=opt.jobflavour, 
+        jobflavour=opt.jobflavour, 
         **opt.htc_arguments
     )
 

tests/unit/test_job_submitter.py

@@ -7,7 +7,7 @@
 import pytest
 
 from pylhc_submitter.job_submitter import main as job_submit
-from pylhc_submitter.submitter.iotools import get_server_from_uri, is_eos_uri, uri_to_path
+from pylhc_submitter.submitter.iotools import uri_to_path
 from pylhc_submitter.utils.environment import on_linux, on_windows
 
 SUBFILE = "queuehtc.sub"
@@ -41,7 +41,6 @@ def test_output_directory(tmp_path):
     )
     setup.create_mask()
     job_submit(**asdict(setup))
-    _test_output(setup)
 
 
 def test_detects_wrong_uri(tmp_path):
@@ -62,7 +61,6 @@ def test_detects_wrong_uri(tmp_path):
         assert "EOS-URI" in str(e)
 
 
-
 @run_only_on_linux
 def test_job_creation_and_localrun_with_multiline_maskstring(tmp_path):
     """ Tests that the jobs are created and can be run locally from a multiline mask-string. """
@@ -84,6 +82,23 @@ def test_job_creation_and_dryrun(tmp_path, maskfile):
     _test_output(setup, post_run=False)
 
 
+@run_only_on_linux
+@pytest.mark.parametrize("maskfile", [True, False])
+@pytest.mark.parametrize("destination", [True, False])
+def test_more_subfile_content(tmp_path, maskfile, destination):
+    """ Tests that the jobs are created as dry-run from mask-file and from mask-string. """
+    setup = InputParameters(
+        jobflavour="espresso",  # change from default
+        working_directory=tmp_path, 
+        dryrun=True,
+        output_destination= tmp_path / "my_new_output" / "long_path" if destination else None,
+        job_output_dir="MyOutputDataHere",  # change from default
+        )
+    setup.create_mask(as_file=maskfile)
+    job_submit(**asdict(setup))
+    _test_subfile_content(setup)
+
+
 @run_only_on_linux
 @pytest.mark.parametrize("maskfile", [True, False])
 def test_find_errorneous_percentage_signs(tmp_path, maskfile):
@@ -243,7 +258,7 @@ def _test_subfile_content(setup: InputParameters):
         if setup.output_destination is None:
             assert filecontents["transfer_output_files"] == setup.job_output_dir
         else:
-            assert "transfer_output_files" not in filecontents
+            assert filecontents["transfer_output_files"] == '""'
 
         for key in setup.htc_arguments.keys():
             assert filecontents[key] == setup.htc_arguments[key]