Merge branch 'main' into file_staging

Author: hategan

.gitignore

@@ -17,4 +17,6 @@ psi_j_python.egg-info/
 venv*
 .venv*
 build/
-.packages/
+.packages/
+docs/.web-build
+web-build/

psij-ci-setup

@@ -217,7 +217,7 @@ check_email() {
 }
 
 check_key() {
-    if ! -f ~/.psij/key ; then
+    if [ ! -f ~/.psij/key ]; then
         BASE_URL=`egrep '^[^#]*server_url.*' ./testing.conf |awk '{split($0, a, "="); print a[2]}'|xargs`
         AUTH_URL="${BASE_URL}/auth.html"
 
@@ -311,6 +311,7 @@ else
     echo "Done"
 fi
 
+export PYTHONPATH=`pwd`/.packages:$PYTHONPATH
 
 HOUR=`echo $(($RANDOM % 24))`
 MINUTE=`echo $(($RANDOM % 60))`

src/psij/executors/batch/batch_scheduler_executor.py

@@ -545,8 +545,12 @@ def _clean_submit_script(self, job: Job) -> None:
             assert isinstance(self.config, BatchSchedulerExecutorConfig)
             if not self.config.keep_files:
                 submit_file_path = self.work_directory / (job.id + '.job')
-                if submit_file_path.exists():
+                try:
                     submit_file_path.unlink()
+                except FileNotFoundError:
+                    # this can reasonably happen for attached jobs when the main
+                    # job cleans up the script instead
+                    pass
         except Exception as ex:
             logger.warning('Job %s: failed clean submit script: %s', job.id, ex)
 
@@ -611,7 +615,10 @@ def _delete_aux_file(self, job: Optional[Job] = None, suffix: Optional[str] = No
             assert suffix
             path = self.work_directory / (job.native_id + suffix)
         if force or path.exists():
-            path.unlink()
+            try:
+                path.unlink()
+            except FileNotFoundError:
+                pass  # see above; attached job may race with original job
 
     def list(self) -> List[str]:
         """Returns a list of jobs known to the underlying implementation.

src/psij/executors/batch/cobalt.py

@@ -36,6 +36,10 @@ class CobaltJobExecutor(BatchSchedulerExecutor):
     monitor, and cancel jobs.
 
     Creates a batch script with #COBALT directives when submitting a job.
+
+    Custom attributes prefixed with `cobalt.` are rendered as long-form directives in the script.
+    For example, setting `custom_attributes['cobalt.m'] = 'co'` results in the `#COBALT --m=co`
+    directive being placed in the submit script.
     """
 
     # see https://Cobalt.schedmd.com/squeue.html

src/psij/executors/batch/lsf.py

@@ -35,6 +35,11 @@ class LsfJobExecutor(BatchSchedulerExecutor):
     monitor, and cancel jobs.
 
     Creates a batch script with #BSUB directives when submitting a job.
+
+    Renders all custom attributes of the form `lsf.<name>` into the corresponding
+    LSF directive. For example, setting
+    `job.spec.attributes.custom_attributes['lsf.core_isolation'] = '0' results in a
+    `#BSUB -core_isolation 0` directive being placed in the submit script.
     """
 
     # see https://www.ibm.com/docs/en/spectrum-lsf/10.1.0?topic=bjobs-description

src/psij/executors/batch/pbs.py

@@ -22,6 +22,6 @@ def __init__(self, url: Optional[str] = None, config: Optional[PBSExecutorConfig
         """
         if not config:
             config = PBSExecutorConfig()
-        generator = TemplatedScriptGenerator(config, Path(__file__).parent / 'pbspro'
+        generator = TemplatedScriptGenerator(config, Path(__file__).parent / 'pbs'
                                              / 'pbspro.mustache')
         super().__init__(generator, url=url, config=config)

src/psij/executors/batch/pbs/pbs_classic.mustache

@@ -10,9 +10,7 @@
 {{/job.spec.inherit_environment}}
 
 {{#job.spec.resources}}
-    {{#process_count}}
-#PBS -l nodes={{job.spec.resources.computed_node_count}}:ppn={{.}}{{#job.spec.resources.gpu_cores_per_process}}:gpus={{.}}{{/job.spec.resources.gpu_cores_per_process}}
-    {{/process_count}}
+#PBS -l nodes={{computed_node_count}}:ppn={{computed_processes_per_node}}{{#gpu_cores_per_process}}:gpus={{.}}{{/gpu_cores_per_process}}
     {{#exclusive_node_use}}
 #PBS -n
     {{/exclusive_node_use}}
@@ -51,8 +49,9 @@ only results in empty files that are not cleaned up}}
 {{#job.spec.inherit_environment}}
 #PBS -V
 {{/job.spec.inherit_environment}}
+
 {{#env}}
-#PBS -v {{name}}={{value}}
+export {{name}}={{value}}
 {{/env}}
 
 exec &>> "{{psij.script_dir}}/$PBS_JOBID.out"

src/psij/executors/batch/pbs/pbspro.mustache

@@ -10,9 +10,7 @@
 {{/job.spec.inherit_environment}}
 
 {{#job.spec.resources}}
-    {{#process_count}}
-#PBS -l select={{job.spec.resources.computed_node_count}}:ncpus={{.}}
-    {{/process_count}}
+#PBS -l select={{computed_node_count}}:ncpus={{computed_processes_per_node}}:mpiprocs={{computed_processes_per_node}}
     {{#exclusive_node_use}}
 #PBS -l place=scatter:exclhost
     {{/exclusive_node_use}}
@@ -56,8 +54,9 @@ exec &>> "{{psij.script_dir}}/$PBS_JOBID.out"
 {{#job.spec.inherit_environment}}
 #PBS -V
 {{/job.spec.inherit_environment}}
+
 {{#env}}
-#PBS -v {{name}}={{value}}
+export {{name}}={{value}}
 {{/env}}
 
 {{> batch_lib}}

src/psij/executors/batch/pbs_base.py

@@ -58,6 +58,10 @@ class GenericPBSJobExecutor(BatchSchedulerExecutor):
     monitor, and cancel jobs.
 
     Creates a batch script with #PBS directives when submitting a job.
+
+    Custom attributes prefixed with `pbs.` are rendered as directives in the script.
+    For example, setting `custom_attributes['pbs.c'] = 'n'` results in the `#PBS -c n`
+    directive being placed in the submit script, which disables checkpointing.
     """
 
     def __init__(self, generator: TemplatedScriptGenerator, url: Optional[str] = None,

src/psij/executors/batch/slurm.py

@@ -28,6 +28,11 @@ class SlurmJobExecutor(BatchSchedulerExecutor):
     monitor, and cancel jobs.
 
     Creates a batch script with #SBATCH directives when submitting a job.
+
+    Renders all custom attributes set on a job's attributes with a `slurm.` prefix into
+    corresponding Slurm directives with long-form parameters. For example,
+    `job.spec.attributes.custom_attributes['slurm.qos'] = 'debug'` causes a directive
+    `#SBATCH --qos=debug` to be placed in the submit script.
     """
 
     # see https://slurm.schedmd.com/squeue.html