HTCondor v25 Fix (#44)

Author: fsoubelet

CHANGELOG.md

@@ -1,5 +1,10 @@
 # `pylhc-submitter` Changelog
 
+## Version 2.0.5
+
+- Implemented compatibility with the recently released `htcondor 25.0` which brought breaking changes.
+- Fixed the display of the quick stats summary message shown right after submitting to HTCondor.
+
 ## Version 2.0.4
 
 - Fixed use of `np.NaN` to ensure compatibility with `numpy 2.0`.
@@ -21,12 +26,12 @@
 - General code cleanup/refactoring/documentation:
   - Partly breaks backward compatibility, if individual methods of the `job_submitter`-functionality have been used.
   - Does not affect any setups simply calling the `main()` function of `job_submitter.py` or calling the `job_submitter` as a module.
-  - Apart from some fixed imports, following the new structure, the `autosix` module has been untouched. 
+  - Apart from some fixed imports, following the new structure, the `autosix` module has been untouched.
 
 
 - New Feature of `job_submitter`:
-  - `output_destination` input parameter, which sets an output directory in which the folder-stucture 
-    for the jobs will be replicated and the job's `job_output_dir` will be copied into "manually" at the end of the job, 
+  - `output_destination` input parameter, which sets an output directory in which the folder-stucture
+    for the jobs will be replicated and the job's `job_output_dir` will be copied into "manually" at the end of the job,
     instead of having the directory transferred back to the `working directory` by htcondor.
 
 ## Version 1.1.1
@@ -38,15 +43,15 @@
 This release adds some changes the `autosix` module:
 
 - New Features:
-  - Added `sixdesk_directory` option, which allows the user to choose their own 
+  - Added `sixdesk_directory` option, which allows the user to choose their own
     sixdesk environment (default is PRO on AFS).
   - Added `max_materialize` option, which allows the user to specify the amount of jobs that
     materialize at once per SixDesk Workspace (i.e. one realization in the `replace_dict`).
     This enables the user to send more jobs to HTCondor than are allowed within their user limit.
     See the HTCondor API for details. This option requires writing rights in the `sixdesk_directory`.
-  - Allow `ENERGY` and `EMITTANCE` to be set via `replace_dict`, which are then 
+  - Allow `ENERGY` and `EMITTANCE` to be set via `replace_dict`, which are then
     passed to the `sixdeskenv` (`GAMMA` is calculated from the `EMITTANCE` automatically).
-    
+
 - Changes:
   - Big object-oriented restructuring of the Stages and increased use of Dataclasses.
   - Some other small changes to improve readablity.

pylhc_submitter/__init__.py

@@ -11,7 +11,7 @@
 __title__ = "pylhc_submitter"
 __description__ = "pylhc-submitter contains scripts to simplify the creation and submission of jobs to HTCondor at CERN"
 __url__ = "https://github.com/pylhc/submitter"
-__version__ = "2.0.4"
+__version__ = "2.0.5"
 __author__ = "pylhc"
 __author_email__ = "pylhc@github.com"
 __license__ = "MIT"

pylhc_submitter/job_submitter.py

@@ -181,16 +181,29 @@
 
 LOG = logging.getLogger(__name__)
 
+# ------------------------------------------------------------------ #
+# Importing htcondor is tricky because they broke the API in v25 LTS #
+# ------------------------------------------------------------------ #
 
 try:
-    import htcondor
+    # First, try HTCondor 25.x API
+    import htcondor2 as htcondor
+
+    LOG.debug("Using htcondor2 bindings (HTCondor 25.x+).")
 except ImportError:
-    platform = "macOS" if sys.platform == "darwin" else "windows"
-    LOG.warning(
-        f"htcondor python bindings are linux-only. You can still use job_submitter on {platform}, "
-        "but only for local runs."
-    )
-    htcondor = None
+    try:
+        # Fallback to previous LTS HTCondor API
+        import htcondor
+
+        LOG.debug("Using htcondor bindings (HTCondor <25).")
+    except ImportError:
+        # Neither available: must be macOS or Windows
+        platform = "macOS" if sys.platform == "darwin" else "windows"
+        LOG.warning(
+            f"htcondor python bindings are linux-only. You can still use job_submitter on {platform}, "
+            "but only for local runs."
+        )
+        htcondor = None
 
 
 def get_params():

pylhc_submitter/submitter/htc_utils.py

@@ -41,14 +41,24 @@
 from pylhc_submitter.submitter.mask import is_mask_file
 from pylhc_submitter.utils.environment import on_windows
 
-try:
-    import htcondor  # noqa: N801
-except ImportError:  # will be handled by job_submitter
-
-    class htcondor:  # noqa: N801
-        """Dummy HTCondor module. To satisfy the typing."""
+# ------------------------------------------------------------------ #
+# Importing htcondor is tricky because they broke the API in v25 LTS #
+# ------------------------------------------------------------------ #
 
-        Submit: Any = None
+try:
+    # First, try HTCondor 25.x API
+    import htcondor2 as htcondor  # noqa: N801
+except ImportError:
+    try:
+        # Fallback to previous LTS HTCondor API
+        import htcondor  # noqa: N801
+    except ImportError:  # will be handled by job_submitter
+        # Neither API is available – define dummy stubs for typing
+
+        class htcondor:  # noqa: N801
+            """Dummy HTCondor module to satisfy typing."""
+
+            Submit: Any = None
 
 
 if TYPE_CHECKING:

pylhc_submitter/submitter/iotools.py

@@ -247,7 +247,7 @@ def get_server_from_uri(path: Path | str) -> str:
 def print_stats(new_jobs: JobNamesType, finished_jobs: JobNamesType):
     """Print some quick statistics."""
     text = [
-        "\n------------- QUICK STATS ----------------"
+        "\n------------- QUICK STATS ----------------",
         f"Jobs total:{len(new_jobs) + len(finished_jobs):d}",
         f"Jobs to run: {len(new_jobs):d}",
         f"Jobs already finished: {len(finished_jobs):d}",

pyproject.toml

@@ -79,10 +79,24 @@ changelog = "https://github.com/pylhc/submitter/blob/master/CHANGELOG.md"
 markers = [
     "cern_network: tests that require access to afs or the technical network",
 ]
+
+addopts = [
+    "--import-mode=importlib",
+]
+
 # Helpful for pytest-debugging (leave commented out on commit):
 # log_cli=true
 # log_level=DEBUG
 
+[tool.coverage.run]
+relative_files = true
+
+[tool.coverage.report]
+exclude_also = [
+  "if TYPE_CHECKING:",  # do not count type checking imports (ignored at runtime) for coverage
+  "except ImportError:",  # do not count missing optional dependencies set to None, we monkeypatch and test that
+]
+
 # ----- Dev Tools Configuration ----- #
 
 [tool.ruff]