Merge branch 'main' into hostlist

Author: hategan

.mypy

@@ -17,3 +17,5 @@ ignore_missing_imports = True
 [mypy-pystache.*]
 ignore_missing_imports = True
 
+[mypy-typing_compat.*]
+ignore_missing_imports = True

docs/api.rst

@@ -87,7 +87,7 @@ terminal:
 
 
 JobExecutor Base Class
-^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^
 
 The ``psij.JobExecutor`` class is abstract, but offers concrete static methods
 for registering, fetching, and listing subclasses of itself.
@@ -216,7 +216,7 @@ multiple
     :noindex:
 
 Other Package Contents
-----------------
+----------------------
 
 .. automodule:: psij.exceptions
     :members:

requirements.txt

@@ -2,3 +2,4 @@ filelock~=3.4
 psutil~=5.9
 pystache>=0.6.0
 typeguard~=2.12
+typing-compat

src/psij/__init__.py

@@ -7,7 +7,7 @@
 from typing import Callable, TypeVar
 
 from psij.descriptor import Descriptor
-from .exceptions import SubmitException, InvalidJobException, UnreachableStateException
+from .exceptions import SubmitException, InvalidJobException
 from .job import Job, JobStatusCallback
 from .job_attributes import JobAttributes
 from .job_executor import JobExecutor
@@ -17,15 +17,14 @@
 from .job_state import JobState
 from .job_status import JobStatus
 from .resource_spec import ResourceSpec, ResourceSpecV1
-from .serialize import Export, Import
 from .version import VERSION
 
 __version__ = VERSION
 
 __all__ = [
     'JobExecutor', 'JobExecutorConfig', 'Job', 'JobStatusCallback', 'JobSpec', 'JobAttributes',
     'JobStatus', 'JobState', 'ResourceSpec', 'ResourceSpecV1', 'Launcher', 'SubmitException',
-    'InvalidJobException', 'UnreachableStateException', 'Export', 'Import'
+    'InvalidJobException', 'UnreachableStateException'
 ]
 
 logger = logging.getLogger(__name__)

src/psij/exceptions.py

@@ -66,26 +66,3 @@ def __init__(self, message: str, exception: Optional[Exception] = None,
         conditions such an error would persist across subsequent re-tries until correct credentials
         are used.
         """
-
-
-class UnreachableStateException(Exception):
-    """
-    Indicates that a job state being waited for cannot be reached.
-
-    This exception is thrown when the :func:`~psij.Job.wait` method is called with a set of
-    states that cannot be reached by the job when the call is made.
-    """
-
-    def __init__(self, status: 'psij.JobStatus') -> None:
-        """
-        Constructs an `UnreachableStateException`.
-
-        :param status: The :class:`~psij.JobStatus` that the job was in when
-            :func:`~psij.Job.wait` was called and which prevents the desired states to be
-            reached.
-        """
-        self.status = status
-        """
-        Returns the job status that has caused an implementation to determine that the desired
-        states passed to the :func:`~psij.Job.wait` method cannot be reached.
-        """

src/psij/executors/batch/batch_scheduler_executor.py

@@ -7,7 +7,7 @@
 from datetime import timedelta
 from pathlib import Path
 from threading import Thread, RLock
-from typing import Optional, List, Dict, Collection, cast, TextIO, Union
+from typing import Optional, List, Dict, Collection, cast, Union, IO
 
 from .escape_functions import bash_escape
 from psij.launchers.script_based_launcher import ScriptBasedLauncher
@@ -38,6 +38,8 @@ def _attrs_to_mustache(job: Job) -> Dict[str, Union[object, List[Dict[str, objec
     for k, v in job.spec.attributes._custom_attributes.items():
         ks = k.split('.', maxsplit=1)
         if len(ks) == 2:
+            # always use lower case here
+            ks[0] = ks[0].lower()
             if ks[0] not in r:
                 r[ks[0]] = []
             cast(List[Dict[str, object]], r[ks[0]]).append({'key': ks[1], 'value': v})
@@ -272,7 +274,7 @@ def attach(self, job: Job, native_id: str) -> None:
 
     @abstractmethod
     def generate_submit_script(self, job: Job, context: Dict[str, object],
-                               submit_file: TextIO) -> None:
+                               submit_file: IO[str]) -> None:
         """Called to generate a submit script for a job.
 
         Concrete implementations of batch scheduler executors must override this method in

src/psij/executors/batch/cobalt.py

@@ -1,7 +1,7 @@
 """Defines a JobExecutor for the Cobalt resource manager."""
 from datetime import timedelta
 from pathlib import Path
-from typing import Optional, Collection, List, Dict, TextIO
+from typing import Optional, Collection, List, Dict, IO
 import re
 import os
 import stat
@@ -59,7 +59,7 @@ def __init__(
         )
 
     def generate_submit_script(
-        self, job: Job, context: Dict[str, object], submit_file: TextIO
+        self, job: Job, context: Dict[str, object], submit_file: IO[str]
     ) -> None:
         """See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
         self.generator.generate_submit_script(job, context, submit_file)

src/psij/executors/batch/cobalt/cobalt.mustache

@@ -20,14 +20,23 @@
     {{#queue_name}}
 #COBALT --queue={{.}}
     {{/queue_name}}
+    {{!Like PBS, Cobalt uses specially named queues for reservations, so we send the job to the
+    respective queue when a reservation ID is specified.}}
+    {{#reservation_id}}
+#COBALT --queue={{.}}
+    {{/reservation_id}}
     {{#project_name}}
 #COBALT --project={{.}}
     {{/project_name}}
-    {{#custom_attributes.COBALT}}
-#COBALT --{{key}}="{{value}}"
-    {{/custom_attributes.COBALT}}
 {{/job.spec.attributes}}
 
+{{#custom_attributes}}
+    {{#cobalt}}
+#COBALT --{{key}}="{{value}}"
+    {{/cobalt}}
+{{/custom_attributes}}
+
+
 {{#env}}
 #COBALT --env {{name}}={{value}}
 {{/env}}

src/psij/executors/batch/lsf.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import re
 import json
-from typing import Optional, Collection, List, Dict, TextIO
+from typing import Optional, Collection, List, Dict, IO
 
 from psij import Job, JobStatus, JobState, SubmitException
 from psij.executors.batch.batch_scheduler_executor import (
@@ -62,7 +62,7 @@ def __init__(self, url: Optional[str], config: Optional[LsfExecutorConfig] = Non
         )
 
     def generate_submit_script(
-        self, job: Job, context: Dict[str, object], submit_file: TextIO
+        self, job: Job, context: Dict[str, object], submit_file: IO[str]
     ) -> None:
         """See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
         assert job.spec is not None

src/psij/executors/batch/lsf/lsf.mustache

@@ -59,6 +59,13 @@
 
 {{/job.spec.attributes}}
 
+{{#custom_attributes}}
+    {{#lsf}}
+#BSUB -{{key}} "{{value}}"
+    {{/lsf}}
+{{/custom_attributes}}
+
+
 {{!since we redirect the output manually, below, tell LSF not to do its own thing, since it
 only results in empty files that are not cleaned up}}
 #BSUB -e /dev/null