Merge pull request #4 from XENONnT/rundb_surgery

Rundb surgery

Author: rynge

.gitignore

@@ -2,4 +2,5 @@
 .pydevproject
 __pycache__
 *.dbtoken
+outsource.egg-info
 

README.md

@@ -1,12 +1,16 @@
 # outsource
 Job submission code for XENONnT
 
+## Installation
+TODO
+
+Outsource requires modules from [utilix](https://github.com/XENONnT/utilix)
+
 ## Configuration file
 
-This tool expects a configuration file named `$HOME/.xenonnt.conf`. Note that 
-environment variables can be used in the form `$HOME`. Example:
+Just like utilix, this tool expects a configuration file named `$HOME/.xenonnt.conf`. Particularly it uses information in the field of the config with header 'Outsource', see below: 
 
-    [Common]
+    [RunDB]
 
     rundb_api_url = [ask Evan]
     rundb_api_user = [ask Evan]

outsource/Config.py

@@ -1,53 +1,12 @@
-import re
-import os
-import time
-import configparser
-
-dir_raw = '/xenon/xenon1t/raw'
-
-
-class EnvInterpolation(configparser.BasicInterpolation):
-    '''Interpolation which expands environment variables in values.'''
 
-    def before_get(self, parser, section, option, value, defaults):
-        return os.path.expandvars(value)
-
-
-class Config():
-    
-    # singleton
-    instance = None
-    
-    def __init__(self):
-        if not Config.instance:
-            Config.instance = Config.__Config()
-            
+import os
+from utilix.config import Config
 
-    def __getattr__(self, name):
-        return getattr(self.instance, name)
-    
-    
-    class __Config(configparser.ConfigParser):
-        
-        def __init__(self):
-            
-            config_file_path = os.path.join(os.environ['HOME'], '.xenonnt.conf')
-            print('Loading configuration from %s' %(config_file_path)) 
-            
-            configparser.ConfigParser.__init__(self, interpolation=EnvInterpolation())
-            try:
-                self.readfp(open(config_file_path), 'r')
-            except FileNotFoundError as e:
-                raise RuntimeError('Unable to open %s. Please see the README for an example configuration' %(config_file_path)) from e
+config = Config()
 
-        def base_dir(self):
-            return os.path.dirname(__file__)
-        
-        def work_dir(self):
-            return self.get('Outsource', 'work_dir')
-        
-        def runs_dir(self):
-            return os.path.join(self.work_dir(), 'runs')
+dir_raw = '/xenon/xenon1t/raw'
 
-        def pegasus_path(self):
-            return self.get('Outsource', 'pegasus_path')
+base_dir = os.path.dirname(__file__)
+work_dir = config.get('Outsource', 'work_dir')
+runs_dir = os.path.join(work_dir, 'runs')
+pegasus_path = config.get('Outsource', 'pegasus_path')

outsource/Outsource.py

@@ -9,14 +9,13 @@
 import sys
 
 from outsource.Shell import Shell
-from outsource.Config import Config
+from outsource.Config import config, pegasus_path, base_dir, work_dir, runs_dir
 
-config = Config()
 logger = logging.getLogger("outsource")
 
 # Pegasus environment
-sys.path.insert(0, os.path.join(config.pegasus_path(), 'lib64/python2.6/site-packages'))
-os.environ['PATH'] = os.path.join(config.pegasus_path(), 'bin') + ':' + os.environ['PATH']
+sys.path.insert(0, os.path.join(pegasus_path, 'lib64/python2.6/site-packages'))
+os.environ['PATH'] = os.path.join(pegasus_path, 'bin') + ':' + os.environ['PATH']
 from Pegasus.DAX3 import *
 
 
@@ -88,7 +87,7 @@ def submit_workflow(self):
         except OSError:
             pass
         try:
-            os.makedirs(config.runs_dir(), 0o755)
+            os.makedirs(runs_dir, 0o755)
         except OSError:
             pass
 
@@ -109,31 +108,31 @@ def _generate_dax(self):
         dax = ADAG('xenonnt')
 
         # event callouts
-        dax.invoke('start',  config.base_dir() + '/workflow/events/wf-start')
-        dax.invoke('at_end',  config.base_dir() + '/workflow/events/wf-end')
+        dax.invoke('start', base_dir + '/workflow/events/wf-start')
+        dax.invoke('at_end', base_dir + '/workflow/events/wf-end')
 
         # Add executables to the DAX-level replica catalog
         wrapper = Executable(name='run-pax.sh', arch='x86_64', installed=False)
-        wrapper.addPFN(PFN('file://' + config.base_dir() + '/workflow/run-pax.sh', 'local'))
+        wrapper.addPFN(PFN('file://' + base_dir + '/workflow/run-pax.sh', 'local'))
         wrapper.addProfile(Profile(Namespace.PEGASUS, 'clusters.size', 2))
         dax.addExecutable(wrapper)
 
         merge = Executable(name='merge.sh', arch='x86_64', installed=False)
-        merge.addPFN(PFN('file://' + config.base_dir() + '/workflow/merge.sh', 'local'))
+        merge.addPFN(PFN('file://' + base_dir + '/workflow/merge.sh', 'local'))
         dax.addExecutable(merge)
 
         upload = Executable(name='upload.sh', arch='x86_64', installed=False)
-        upload.addPFN(PFN('file://' + config.base_dir() + '/workflow/upload.sh', 'local'))
+        upload.addPFN(PFN('file://' + base_dir + '/workflow/upload.sh', 'local'))
         dax.addExecutable(upload)
 
         # determine_rse - a helper for the job to determine where to pull data from
         determine_rse = File('determine_rse.py')
-        determine_rse.addPFN(PFN('file://' + os.path.join(config.base_dir(), 'workflow/determine_rse.py'), 'local'))
+        determine_rse.addPFN(PFN('file://' + os.path.join(base_dir, 'workflow/determine_rse.py'), 'local'))
         dax.addFile(determine_rse)
 
         # paxify is what processes the data. Gets called by the executable run-pax.sh
         paxify = File('paxify.py')
-        paxify.addPFN(PFN('file://' + os.path.join(config.base_dir(), 'workflow/paxify.py'), 'local'))
+        paxify.addPFN(PFN('file://' + os.path.join(base_dir, 'workflow/paxify.py'), 'local'))
         dax.addFile(paxify)
 
         for dbcfg in self._dbcfgs:
@@ -225,9 +224,9 @@ def _generate_dax(self):
             upload_job = Job("upload.sh")
             upload_job.addProfile(Profile(Namespace.HINTS, 'execution.site', 'local'))
             upload_job.uses(merged_root, link=Link.INPUT)
-            upload_job.addArguments(dbcfg.name, 
+            upload_job.addArguments(dbcfg.name,
                                     merged_root,
-                                    config.base_dir())
+                                    base_dir)
             dax.addJob(upload_job)
             dax.depends(parent=merge_job, child=upload_job)
 
@@ -242,23 +241,23 @@ def _plan_and_submit(self):
         Call out to plan-env-helper.sh to start the workflow
         '''
 
-        cmd = ' '.join([os.path.join(config.base_dir(), 'workflow/plan-env-helper.sh'),
-                        config.pegasus_path(),
-                        config.base_dir(),
-                        config.work_dir(),
+        cmd = ' '.join([os.path.join(base_dir, 'workflow/plan-env-helper.sh'),
+                        pegasus_path,
+                        base_dir,
+                        work_dir,
                         self._generated_dir(),
-                        config.runs_dir(),
+                        runs_dir,
                         self._wf_id])
         shell = Shell(cmd, log_cmd = False, log_outerr = True)
         shell.run()
 
 
     def _generated_dir(self):
-        return os.path.join(config.work_dir(), 'generated', self._wf_id)
+        return os.path.join(work_dir, 'generated', self._wf_id)
 
 
     def _workflow_dir(self):
-        return os.path.join(config.runs_dir(), self._wf_id)
+        return os.path.join(runs_dir, self._wf_id)
 
 
     def _validate_x509_proxy(self):

outsource/RunConfig.py

@@ -1,10 +1,9 @@
 import os
 import re
 import time
-from outsource.Config import Config
-from outsource import db
+from .Config import config, base_dir, work_dir
+from utilix import db
 
-config = Config()
 
 
 class RunConfigBase:
@@ -13,8 +12,8 @@ class RunConfigBase:
     _update_run_db = False
     _force_rerun = False
     _x509_proxy = os.path.join(os.environ['HOME'], 'user_cert')
-    _executable = os.path.join(config.base_dir(), 'workflow', 'run-pax.sh')
-    _workdir = config.get('Outsource', 'work_dir')
+    _executable = os.path.join(base_dir, 'workflow', 'run-pax.sh')
+    _workdir = work_dir
     _workflow_id = re.sub('\..*', '', str(time.time()))
     _pax_version = config.get('Outsource', 'pax_version')
 

outsource/__init__.py

@@ -1,4 +0,0 @@
-from outsource.rundb import DB
-
-# initialize instance of DB here since we just need one instance for all modules to use
-db = DB()

outsource/rundb.py

@@ -2,7 +2,7 @@
 
 
 def readme():
-    with open('README.rst') as f:
+    with open('README.md') as f:
         return f.read()
 
 
@@ -12,8 +12,8 @@ def readme():
       long_description=readme(),
       url='https://github.com/XENONnT/outsource',
       packages=['outsource'],
-      install_requires=[
-          'markdown',
-      ],
+      install_requires=['markdown',
+                        'utilix'
+                        ],
       include_package_data=True,
       zip_safe=False)