Merge remote-tracking branch 'upstream/pr/100' into dev

Conflicts:
	batchspawner/batchspawner.py

Author: rkdarst

batchspawner/batchspawner.py

@@ -17,6 +17,7 @@
 """
 import pwd
 import os
+import re
 
 import xml.etree.ElementTree as ET
 
@@ -46,7 +47,7 @@ def format_template(template, *args, **kwargs):
     """
     if isinstance(template, Template):
         return template.render(*args, **kwargs)
-    elif  '{{' in template or '{%' in template:
+    elif '{{' in template or '{%' in template:
         return Template(template).render(*args, **kwargs)
     return template.format(*args, **kwargs)
 
@@ -78,52 +79,52 @@ class BatchSpawnerBase(Spawner):
     # override default server ip since batch jobs normally running remotely
     ip = Unicode("0.0.0.0", help="Address for singleuser server to listen at").tag(config=True)
 
-    exec_prefix = Unicode('sudo -E -u {username}', \
+    exec_prefix = Unicode('sudo -E -u {username}',
         help="Standard executon prefix (e.g. the default sudo -E -u {username})"
         ).tag(config=True)
 
     # all these req_foo traits will be available as substvars for templated strings
-    req_queue = Unicode('', \
+    req_queue = Unicode('',
         help="Queue name to submit job to resource manager"
         ).tag(config=True)
 
-    req_host = Unicode('', \
+    req_host = Unicode('',
         help="Host name of batch server to submit job to resource manager"
         ).tag(config=True)
 
-    req_memory = Unicode('', \
+    req_memory = Unicode('',
         help="Memory to request from resource manager"
         ).tag(config=True)
 
-    req_nprocs = Unicode('', \
+    req_nprocs = Unicode('',
         help="Number of processors to request from resource manager"
         ).tag(config=True)
 
-    req_ngpus = Unicode('', \
+    req_ngpus = Unicode('',
         help="Number of GPUs to request from resource manager"
         ).tag(config=True)
 
-    req_runtime = Unicode('', \
+    req_runtime = Unicode('',
         help="Length of time for submitted job to run"
         ).tag(config=True)
 
-    req_partition = Unicode('', \
+    req_partition = Unicode('',
         help="Partition name to submit job to resource manager"
         ).tag(config=True)
 
-    req_account = Unicode('', \
+    req_account = Unicode('',
         help="Account name string to pass to the resource manager"
         ).tag(config=True)
 
-    req_options = Unicode('', \
+    req_options = Unicode('',
         help="Other options to include into job submission script"
         ).tag(config=True)
 
-    req_prologue = Unicode('', \
+    req_prologue = Unicode('',
         help="Script to run before single user server starts."
         ).tag(config=True)
 
-    req_epilogue = Unicode('', \
+    req_epilogue = Unicode('',
         help="Script to run after single user server ends."
         ).tag(config=True)
 
@@ -148,8 +149,7 @@ def _req_keepvars_default(self):
               "added to the defaults in keepvars, "
               "comma separated list.")
 
-
-    batch_script = Unicode('', \
+    batch_script = Unicode('',
         help="Template for job submission script. Traits on this class named like req_xyz "
              "will be substituted in the template for {xyz} using string.Formatter. "
              "Must include {cmd} which will be replaced with the jupyterhub-singleuser command line."
@@ -174,7 +174,7 @@ def get_req_subvars(self):
             subvars['keepvars'] += ',' + subvars['keepvars_extra']
         return subvars
 
-    batch_submit_cmd = Unicode('', \
+    batch_submit_cmd = Unicode('',
         help="Command to run to submit batch scripts. Formatted using req_xyz traits as {xyz}."
         ).tag(config=True)
 
@@ -244,7 +244,7 @@ def submit_batch_script(self):
         return self.job_id
 
     # Override if your batch system needs something more elaborate to read the job status
-    batch_query_cmd = Unicode('', \
+    batch_query_cmd = Unicode('',
         help="Command to run to read job status. Formatted using req_xyz traits as {xyz} "
              "and self.job_id as {job_id}."
         ).tag(config=True)
@@ -335,7 +335,7 @@ def poll(self):
             self.clear_state()
             return 1
 
-    startup_poll_interval = Float(0.5, \
+    startup_poll_interval = Float(0.5,
         help="Polling interval (seconds) to check job state during startup"
         ).tag(config=True)
 
@@ -345,8 +345,9 @@ def start(self):
         if self.user and self.user.server and self.user.server.port:
             self.port = self.user.server.port
             self.db.commit()
-        elif (jupyterhub.version_info < (0,7) and not self.user.server.port)  or \
-             (jupyterhub.version_info >= (0,7) and not self.port):
+        elif (jupyterhub.version_info < (0,7) and not self.user.server.port) or (
+             jupyterhub.version_info >= (0,7) and not self.port
+        ):
             self.port = random_port()
             self.db.commit()
         job = yield self.submit_batch_script()
@@ -367,8 +368,8 @@ def start(self):
                 else:
                     self.log.warn('Job ' + self.job_id + ' neither pending nor running.\n' +
                         self.job_status)
-                    raise RuntimeError('The Jupyter batch job has disappeared '
-                           ' while pending in the queue or died immediately '
+                    raise RuntimeError('The Jupyter batch job has disappeared'
+                           ' while pending in the queue or died immediately'
                            ' after starting.')
             yield gen.sleep(self.startup_poll_interval)
 
@@ -405,7 +406,6 @@ def stop(self, now=False):
                              self.job_id, self.current_ip, self.port)
                 )
 
-import re
 
 class BatchSpawnerRegexStates(BatchSpawnerBase):
     """Subclass of BatchSpawnerBase that uses config-supplied regular expressions
@@ -442,13 +442,15 @@ def state_ispending(self):
         assert self.state_pending_re, "Misconfigured: define state_running_re"
         if self.job_status and re.search(self.state_pending_re, self.job_status):
             return True
-        else: return False
+        else:
+            return False
 
     def state_isrunning(self):
         assert self.state_running_re, "Misconfigured: define state_running_re"
         if self.job_status and re.search(self.state_running_re, self.job_status):
             return True
-        else: return False
+        else:
+            return False
 
     def state_gethost(self):
         assert self.state_exechost_re, "Misconfigured: define state_exechost_re"
@@ -461,6 +463,7 @@ def state_gethost(self):
         else:
             return match.expand(self.state_exechost_exp)
 
+
 class TorqueSpawner(BatchSpawnerRegexStates):
     batch_script = Unicode("""#!/bin/sh
 #PBS -q {queue}@{host}
@@ -486,6 +489,7 @@ class TorqueSpawner(BatchSpawnerRegexStates):
     state_running_re = Unicode(r'<job_state>R</job_state>').tag(config=True)
     state_exechost_re = Unicode(r'<exec_host>((?:[\w_-]+\.?)+)/\d+').tag(config=True)
 
+
 class MoabSpawner(TorqueSpawner):
     # outputs job id string
     batch_submit_cmd = Unicode('msub').tag(config=True)
@@ -496,6 +500,7 @@ class MoabSpawner(TorqueSpawner):
     state_running_re = Unicode(r'State="Running"').tag(config=True)
     state_exechost_re = Unicode(r'AllocNodeList="([^\r\n\t\f :"]*)').tag(config=True)
 
+
 class UserEnvMixin:
     """Mixin class that computes values for USER, SHELL and HOME in the environment passed to
     the job submission subprocess in case the batch system needs these for the batch script."""
@@ -522,23 +527,8 @@ def get_env(self):
         env = self.user_env(env)
         return env
 
-class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates):
-    """A Spawner that just uses Popen to start local processes."""
-
-    # all these req_foo traits will be available as substvars for templated strings
-    req_cluster = Unicode('', \
-        help="Cluster name to submit job to resource manager"
-        ).tag(config=True)
-
-    req_qos = Unicode('', \
-        help="QoS name to submit job to resource manager"
-        ).tag(config=True)
-
-    req_srun = Unicode('srun',
-        help="Job step wrapper, default 'srun'.  Set to '' you do not want "
-             "to run in job step (affects environment handling)"
-        ).tag(config=True)
 
+class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates):
     batch_script = Unicode("""#!/bin/bash
 #SBATCH --output={{homedir}}/jupyterhub_slurmspawner_%j.log
 #SBATCH --job-name=spawner-jupyterhub
@@ -558,6 +548,21 @@ class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates):
 echo "jupyterhub-singleuser ended gracefully"
 {{epilogue}}
 """).tag(config=True)
+
+    # all these req_foo traits will be available as substvars for templated strings
+    req_cluster = Unicode('',
+        help="Cluster name to submit job to resource manager"
+        ).tag(config=True)
+
+    req_qos = Unicode('',
+        help="QoS name to submit job to resource manager"
+        ).tag(config=True)
+
+    req_srun = Unicode('srun',
+        help="Job step wrapper, default 'srun'.  Set to '' you do not want "
+             "to run in job step (affects environment handling)"
+        ).tag(config=True)
+
     # outputs line like "Submitted batch job 209"
     batch_submit_cmd = Unicode('sbatch --parsable').tag(config=True)
     # outputs status and exec node like "RUNNING hostname"
@@ -579,6 +584,7 @@ def parse_job_id(self, output):
             raise e
         return id
 
+
 class MultiSlurmSpawner(SlurmSpawner):
     '''When slurm has been compiled with --enable-multiple-slurmd, the
        administrator sets the name of the slurmd instance via the slurmd -N
@@ -591,6 +597,7 @@ def state_gethost(self):
         host = SlurmSpawner.state_gethost(self)
         return self.daemon_resolver.get(host, host)
 
+
 class GridengineSpawner(BatchSpawnerBase):
     batch_script = Unicode("""#!/bin/bash
 #$ -j yes
@@ -640,6 +647,7 @@ def state_gethost(self):
         self.log.error("Spawner unable to match host addr in job {0} with status {1}".format(self.job_id, self.job_status))
         return
 
+
 class CondorSpawner(UserEnvMixin,BatchSpawnerRegexStates):
     batch_script = Unicode("""
 Executable = /bin/sh
@@ -677,6 +685,7 @@ def parse_job_id(self, output):
     def cmd_formatted_for_batch(self):
         return super(CondorSpawner,self).cmd_formatted_for_batch().replace('"','""').replace("'","''")
 
+
 class LsfSpawner(BatchSpawnerBase):
     '''A Spawner that uses IBM's Platform Load Sharing Facility (LSF) to launch notebooks.'''
 
@@ -723,7 +732,6 @@ def state_isrunning(self):
         if self.job_status:
             return self.job_status.split(' ')[0].upper() == 'RUN'
 
-
     def state_gethost(self):
         if self.job_status:
             return self.job_status.split(' ')[1].strip()