Skip to content

Commit 286dc23

Browse files
hateganhategan
authored
Merge pull request #419 from ExaWorks/test_submit_script_generation_basics
Test submit script generation basics
2 parents 0ce67a9 + 143d1ec commit 286dc23

File tree

13 files changed

+132
-34
lines changed

13 files changed

+132
-34
lines changed

‎src/psij/executors/batch/batch_scheduler_executor.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from datetime import timedelta
88
from pathlib import Path
99
from threading import Thread, RLock
10-
from typing import Optional, List, Dict, Collection, cast, TextIO, Union
10+
from typing import Optional, List, Dict, Collection, cast, Union, IO
1111

1212
from .escape_functions import bash_escape
1313
from psij.launchers.script_based_launcher import ScriptBasedLauncher
@@ -38,6 +38,8 @@ def _attrs_to_mustache(job: Job) -> Dict[str, Union[object, List[Dict[str, objec
3838
for k, v in job.spec.attributes._custom_attributes.items():
3939
ks = k.split('.', maxsplit=1)
4040
if len(ks) == 2:
41+
# always use lower case here
42+
ks[0] = ks[0].lower()
4143
if ks[0] not in r:
4244
r[ks[0]] = []
4345
cast(List[Dict[str, object]], r[ks[0]]).append({'key': ks[1], 'value': v})
@@ -268,7 +270,7 @@ def attach(self, job: Job, native_id: str) -> None:
268270

269271
@abstractmethod
270272
def generate_submit_script(self, job: Job, context: Dict[str, object],
271-
submit_file: TextIO) -> None:
273+
submit_file: IO[str]) -> None:
272274
"""Called to generate a submit script for a job.
273275
274276
Concrete implementations of batch scheduler executors must override this method in

‎src/psij/executors/batch/cobalt.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Defines a JobExecutor for the Cobalt resource manager."""
22
from datetime import timedelta
33
from pathlib import Path
4-
from typing import Optional, Collection, List, Dict, TextIO
4+
from typing import Optional, Collection, List, Dict, IO
55
import re
66
import os
77
import stat
@@ -59,7 +59,7 @@ def __init__(
5959
)
6060

6161
def generate_submit_script(
62-
self, job: Job, context: Dict[str, object], submit_file: TextIO
62+
self, job: Job, context: Dict[str, object], submit_file: IO[str]
6363
) -> None:
6464
"""See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
6565
self.generator.generate_submit_script(job, context, submit_file)

‎src/psij/executors/batch/cobalt/cobalt.mustache‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,23 @@
2020
{{#queue_name}}
2121
#COBALT --queue={{.}}
2222
{{/queue_name}}
23+
{{!Like PBS, Cobalt uses specially named queues for reservations, so we send the job to the
24+
respective queue when a reservation ID is specified.}}
25+
{{#reservation_id}}
26+
#COBALT --queue={{.}}
27+
{{/reservation_id}}
2328
{{#project_name}}
2429
#COBALT --project={{.}}
2530
{{/project_name}}
26-
{{#custom_attributes.COBALT}}
27-
#COBALT --{{key}}="{{value}}"
28-
{{/custom_attributes.COBALT}}
2931
{{/job.spec.attributes}}
3032

33+
{{#custom_attributes}}
34+
{{#cobalt}}
35+
#COBALT --{{key}}="{{value}}"
36+
{{/cobalt}}
37+
{{/custom_attributes}}
38+
39+
3140
{{#env}}
3241
#COBALT --env {{name}}={{value}}
3342
{{/env}}

‎src/psij/executors/batch/lsf.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from pathlib import Path
44
import re
55
import json
6-
from typing import Optional, Collection, List, Dict, TextIO
6+
from typing import Optional, Collection, List, Dict, IO
77

88
from psij import Job, JobStatus, JobState, SubmitException
99
from psij.executors.batch.batch_scheduler_executor import (
@@ -62,7 +62,7 @@ def __init__(self, url: Optional[str], config: Optional[LsfExecutorConfig] = Non
6262
)
6363

6464
def generate_submit_script(
65-
self, job: Job, context: Dict[str, object], submit_file: TextIO
65+
self, job: Job, context: Dict[str, object], submit_file: IO[str]
6666
) -> None:
6767
"""See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
6868
assert job.spec is not None

‎src/psij/executors/batch/lsf/lsf.mustache‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,13 @@
5959

6060
{{/job.spec.attributes}}
6161

62+
{{#custom_attributes}}
63+
{{#lsf}}
64+
#BSUB -{{key}} "{{value}}"
65+
{{/lsf}}
66+
{{/custom_attributes}}
67+
68+
6269
{{!since we redirect the output manually, below, tell LSF not to do its own thing, since it
6370
only results in empty files that are not cleaned up}}
6471
#BSUB -e /dev/null

‎src/psij/executors/batch/pbspro.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from datetime import timedelta
22
from pathlib import Path
3-
from typing import Optional, Collection, List, Dict, TextIO
3+
from typing import Optional, Collection, List, Dict, IO
44

55
from psij import Job, JobStatus, JobState, SubmitException
66
from psij.executors.batch.batch_scheduler_executor import BatchSchedulerExecutor, \
@@ -72,7 +72,7 @@ def __init__(self, url: Optional[str] = None, config: Optional[PBSProExecutorCon
7272
# Submit methods
7373

7474
def generate_submit_script(self, job: Job, context: Dict[str, object],
75-
submit_file: TextIO) -> None:
75+
submit_file: IO[str]) -> None:
7676
"""See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
7777
self.generator.generate_submit_script(job, context, submit_file)
7878

‎src/psij/executors/batch/pbspro/pbspro.mustache‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,25 @@
1616
{{/formatted_job_duration}}
1717

1818
{{#job.spec.attributes}}
19-
{{#projectName}}
19+
{{#project_name}}
2020
#PBS -P {{.}}
21-
{{/projectName}}
22-
{{#queueName}}
21+
{{/project_name}}
22+
{{#queue_name}}
2323
#PBS -q {{.}}
24-
{{/queueName}}
25-
{{#custom_attributes.pbs}}
24+
{{/queue_name}}
25+
{{!PBS uses specially named queues for reservations, so we send the job to the respective
26+
queue when a reservation ID is specified.}}
27+
{{#reservation_id}}
28+
#PBS -q {{.}}
29+
{{/reservation_id}}
30+
{{/job.spec.attributes}}
31+
32+
{{#custom_attributes}}
33+
{{#pbs}}
2634
#PBS -{{key}} "{{value}}"
27-
{{/custom_attributes.pbs}}
35+
{{/pbs}}
36+
{{/custom_attributes}}
2837

29-
{{/job.spec.attributes}}
3038

3139
{{!since we redirect the output manually, below, tell pbs not to do its own thing, since it
3240
only results in empty files that are not cleaned up}}

‎src/psij/executors/batch/script_generator.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import pathlib
22
from abc import ABC
3-
from typing import TextIO, Dict, Callable
3+
from typing import Dict, Callable, IO
44

55
import pystache
66

@@ -29,7 +29,7 @@ def __init__(self, config: JobExecutorConfig) -> None:
2929
"""
3030
self.config = config
3131

32-
def generate_submit_script(self, job: Job, context: Dict[str, object], out: TextIO) -> None:
32+
def generate_submit_script(self, job: Job, context: Dict[str, object], out: IO[str]) -> None:
3333
"""Generates a job submit script.
3434
3535
Concerete implementations of submit script generators must implement this method. Its
@@ -76,7 +76,7 @@ def __init__(self, config: JobExecutorConfig, template_path: pathlib.Path,
7676
self.template = pystache.parse(template_file.read())
7777
self.renderer = pystache.Renderer(escape=escape)
7878

79-
def generate_submit_script(self, job: Job, context: Dict[str, object], out: TextIO) -> None:
79+
def generate_submit_script(self, job: Job, context: Dict[str, object], out: IO[str]) -> None:
8080
"""See :func:`~SubmitScriptGenerator.generate_submit_script`.
8181
8282
Renders a submit script using the template specified when this generator was constructed.

‎src/psij/executors/batch/slurm.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from datetime import timedelta
22
from pathlib import Path
3-
from typing import Optional, Collection, List, Dict, TextIO
3+
from typing import Optional, Collection, List, Dict, IO
44

55
from psij import Job, JobStatus, JobState, SubmitException
66
from psij.executors.batch.batch_scheduler_executor import BatchSchedulerExecutor, \
@@ -117,7 +117,7 @@ def __init__(self, url: Optional[str] = None, config: Optional[SlurmExecutorConf
117117
/ 'slurm.mustache')
118118

119119
def generate_submit_script(self, job: Job, context: Dict[str, object],
120-
submit_file: TextIO) -> None:
120+
submit_file: IO[str]) -> None:
121121
"""See :meth:`~.BatchSchedulerExecutor.generate_submit_script`."""
122122
self.generator.generate_submit_script(job, context, submit_file)
123123

‎src/psij/executors/batch/slurm/slurm.mustache‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,13 @@
6161
{{#reservation_id}}
6262
#SBATCH --reservation="{{.}}"
6363
{{/reservation_id}}
64+
{{/job.spec.attributes}}
6465

65-
{{#custom_attributes.slurm}}
66+
{{#custom_attributes}}
67+
{{#slurm}}
6668
#SBATCH --{{key}}="{{value}}"
67-
{{/custom_attributes.slurm}}
68-
69-
{{/job.spec.attributes}}
69+
{{/slurm}}
70+
{{/custom_attributes}}
7071

7172
{{!since we redirect the output manually, below, tell slurm not to do its own thing, since it
7273
only results in empty files that are not cleaned up}}

0 commit comments

Comments
 (0)