don't modify the umask so much with the job.sh file

Author: stefdoerr

jobqueues/celeryfiles/tasks.py

@@ -81,6 +81,7 @@ def _createJobScript(
     fname, workdir, runsh, deviceid, sentinel, datadir, copyextensions
 ):
     import os
+    from jobqueues.util import _makeExecutable
 
     with open(fname, "w") as f:
         f.write("#!/bin/bash\n\n")
@@ -105,4 +106,4 @@ def _createJobScript(
             if os.path.abspath(odir) != os.path.abspath(workdir):
                 f.write("\nmv {} {}".format(" ".join(copyextensions), odir))
 
-    os.chmod(fname, 0o700)
+    _makeExecutable(fname)

jobqueues/localqueue.py

@@ -112,6 +112,8 @@ def run_job(self, deviceid):
         logger.info("Shutting down worker thread")
 
     def _createJobScript(self, fname, workdir, runsh, gpudevice=None):
+        from jobqueues.util import _makeExecutable
+
         with open(fname, "w") as f:
             f.write("#!/bin/bash\n\n")
             # Trap kill signals to create sentinel file
@@ -137,7 +139,7 @@ def _createJobScript(self, fname, workdir, runsh, gpudevice=None):
                 if os.path.abspath(odir) != os.path.abspath(workdir):
                     f.write("\nmv {} {}".format(" ".join(self.copy), odir))
 
-        os.chmod(fname, 0o700)
+        _makeExecutable(fname)
 
     def _setRunning(self, path):
         self._states[path] = "R"
@@ -433,7 +435,7 @@ def _getmemory(self):
         from math import floor
         import psutil
 
-        memory = psutil.virtual_memory().total / 1024 ** 2
+        memory = psutil.virtual_memory().total / 1024**2
         memory *= max(0, min(1, self.ncpu / psutil.cpu_count()))  # Clamp to [0, 1]
         memory = int(floor(memory))
 

jobqueues/lsfqueue.py

@@ -215,7 +215,7 @@ def _find_binary(binary):
         return ret
 
     def _createJobScript(self, fname, workdir, runsh):
-        from jobqueues.util import ensurelist
+        from jobqueues.util import ensurelist, _makeExecutable
 
         workdir = os.path.abspath(workdir)
         with open(fname, "w") as f:
@@ -279,7 +279,7 @@ def _createJobScript(self, fname, workdir, runsh):
                 os.makedirs(datadir, exist_ok=True)
                 f.write(f"\nmv *.{self.trajext} {datadir}")
 
-        os.chmod(fname, 0o700)
+        _makeExecutable(fname)
 
     def retrieve(self):
         # Nothing to do

jobqueues/pbsqueue.py

@@ -142,6 +142,8 @@ def _find_binary(binary):
         return ret
 
     def _createJobScript(self, fname, workdir, runsh):
+        from jobqueues.util import _makeExecutable
+
         workdir = os.path.abspath(workdir)
         if not self.queue and self.ngpu > 0:
             self.queue = "gpgpu"
@@ -186,7 +188,7 @@ def _createJobScript(self, fname, workdir, runsh):
                 os.makedirs(datadir, exist_ok=True)
                 f.write(f"\nmv *.{self.trajext} {datadir}")
 
-        os.chmod(fname, 0o700)
+        _makeExecutable(fname)
 
     def retrieve(self):
         # Nothing to do

jobqueues/sgequeue.py

@@ -185,6 +185,7 @@ def _find_binary(binary):
 
     def _createJobScript(self, fname, workdir, runsh):
         from jobqueues.config import template_env
+        from jobqueues.util import _makeExecutable
 
         workdir = os.path.abspath(workdir)
         sentinel = os.path.normpath(os.path.join(workdir, self._sentinel))
@@ -223,7 +224,7 @@ def _createJobScript(self, fname, workdir, runsh):
         with open(fname, "w") as f:
             f.write(job_str)
 
-        os.chmod(fname, 0o700)
+        _makeExecutable(fname)
 
     def retrieve(self):
         # Nothing to do

jobqueues/slurmqueue.py

@@ -327,6 +327,7 @@ def _find_binary(binary, permissive=False):
 
     def _createJobScript(self, fname, workdir, runsh, nvidia_mps=False, commands=None):
         from jobqueues.config import template_env
+        from jobqueues.util import _makeExecutable
 
         # Create a list of lists with the directory of the run.sh and the run.sh itself
         runsh = ensurelist(runsh)
@@ -399,7 +400,7 @@ def _createJobScript(self, fname, workdir, runsh, nvidia_mps=False, commands=Non
         )
         with open(fname, "w") as f:
             f.write(job_str)
-        os.chmod(fname, 0o700)
+        _makeExecutable(fname)
 
     def retrieve(self):
         # Nothing to do

jobqueues/util.py

@@ -62,3 +62,10 @@ def _filterVisibleGPUdevices(devices, _logger):
         # Only keep the selected visible devices. intersect of the two lists
         devices = [dd for dd in devices if dd in visible_devices]
     return devices
+
+
+def _makeExecutable(fname):
+    import os
+
+    current_mode = os.stat(fname).st_mode
+    os.chmod(fname, current_mode | 0o100)

tests/test_slurmqueue.py

@@ -23,11 +23,13 @@ def datadir(tmpdir, request):
 
 
 def _create_execdir(tmpdir, runscript="run.sh"):
+    from jobqueues.util import _makeExecutable
+
     os.makedirs(tmpdir)
     run_sh = os.path.join(tmpdir, runscript)
     with open(run_sh, "w") as f:
         f.write("sleep 5\n")
-    os.chmod(run_sh, 0o700)
+    _makeExecutable(run_sh)
     return tmpdir