Updated executor handling and back up conf file if updating.

Author: hategan

.flake8

@@ -78,5 +78,5 @@ ignore = B902, D205, D400, D401, D100, W503
 
 per-file-ignores =
     tests/*: D103
-    tests/installer/*: D101, D102, D103, D104, D107
+    tests/installer/*: D101, D102, D103, D104, D105, D107
     tests/installer/main.py: E402, D101, D102, D103, D107

tests/installer/main.py

@@ -235,14 +235,21 @@ async def _confirm_quit(self) -> None:
 
     async def action_quit(self) -> None:
         install_method = self.state.install_method
+        messages = []
+        if self.state.conf_backed_up:
+            messages.append('Your previous configuration was saved in "testing.conf.bk"')
         if install_method is not None and install_method.name == 'custom':
-            self.exit(message=f'Tests can be run with the following command:\n'
-                              f'\t{install_method.preview}')
+            messages.append(f'Tests can be run with the following command:\n'
+                            f'\t{install_method.preview}')
+        if len(messages) > 0:
+            self.exit(message='\n\n'.join(messages))
         else:
             self.exit()
 
     def set_default_executor(self) -> None:
-        label, name = self.state.get_executor()
+        label, name = self.state.get_batch_executor()
+        if name is None:
+            name = 'none'
 
         batch_warner = self.get_widget_by_id('warn-no-batch')
         if name == 'none':

tests/installer/panels/batch_scheduler_panel.py

@@ -6,7 +6,7 @@
 from .panel import Panel
 from ..dialogs import TestJobsDialog
 from ..log import log
-from ..state import Attr
+from ..state import Attr, State
 from ..widgets import MSelect, ShortcutButton
 
 from textual import on
@@ -220,6 +220,10 @@ class BatchSchedulerPanel(Panel):
         ('alt+t', 'toggle_test_job')
     ]
 
+    def __init__(self, state: State) -> None:
+        super().__init__(state)
+        self._auto_scheduler: Optional[str] = None
+
     def _build_widgets(self) -> Widget:
         return Vertical(
             Label('Select and configure a batch system.', classes='header'),
@@ -258,7 +262,7 @@ def _build_widgets(self) -> Widget:
                     Input(id='mqueue-input'),
                     classes='bs-col-2 form-row batch-valid'
                 ),
-                Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job',
+                Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', value=False,
                          id='cb-run-test-job', classes='bs-col-3 m-t-1 batch-valid'),
                 classes='w-100 form-row', id='batch-system-group-2'
             ),
@@ -278,7 +282,11 @@ def _build_widgets(self) -> Widget:
         )
 
     async def validate(self) -> bool:
-        if self.state.run_test_job and self.state.scheduler is not None:
+        run_test_job = self.get_widget_by_id('cb-run-test-job')
+        assert isinstance(run_test_job, Checkbox)
+        scheduler = self._get_scheduler()
+
+        if run_test_job.value and scheduler != 'none' and scheduler != 'local':
             return await self.run_test_jobs()
         else:
             return True
@@ -291,14 +299,19 @@ def label(self) -> str:
     def name(self) -> str:
         return 'scheduler'
 
-    async def activate(self) -> None:
-        self.update_attrs()
+    def _get_scheduler(self) -> str:
         selector = self.get_widget_by_id('batch-selector')
         assert isinstance(selector, Select)
-        sched = selector.selection
-        if sched is None or sched == 'none':
-            selector.focus(False)
-        elif sched == 'local':
+        value = selector.selection
+        assert isinstance(value, str)
+        return value
+
+    async def activate(self) -> None:
+        self.update_attrs()
+        scheduler = self._get_scheduler()
+        if scheduler is None or scheduler == 'none':
+            self.get_widget_by_id('batch-selector').focus(False)
+        elif scheduler == 'local':
             self.app._focus_next()  # type: ignore
         else:
             self.get_widget_by_id('account-input').focus(False)
@@ -316,40 +329,48 @@ def update_attrs(self) -> None:
 
     @on(Select.Changed, '#batch-selector')
     def batch_system_selected(self) -> None:
-        selector = cast(Select[str], self.get_widget_by_id('batch-selector'))
-        sched = selector.selection
-        disabled = (sched is None or sched == 'none' or sched == 'local')
-        if disabled:
-            self.state.scheduler = None
-        else:
-            self.state.scheduler = sched
+        scheduler = self._get_scheduler()
+
+        self.state.set_batch_executor(scheduler)
+
+        disabled = (scheduler is None or scheduler == 'none' or scheduler == 'local')
         for widget in self.query('.batch-valid'):
             widget.disabled = disabled
-        if sched == 'local':
+        run_test_job = self.get_widget_by_id('cb-run-test-job')
+        assert isinstance(run_test_job, Checkbox)
+        run_test_job.value = not disabled
+        if scheduler == 'local':
             self.app._focus_next()  # type: ignore
         else:
             self.get_widget_by_id('account-input').focus(False)
 
     def set_scheduler(self, name: str) -> None:
+        if name == '':
+            name = 'none'
         selector = self.get_widget_by_id('batch-selector')
         assert isinstance(selector, Select)
         selector.value = name
+        self._auto_scheduler = name
 
     @on(Input.Submitted, '#account-input')
-    def account_submitted(self) -> None:
+    def account_submitted(self, event: Input.Submitted) -> None:
+        self.state.update_conf('account', event.value)
         next = self.get_widget_by_id('queue-input')
         next.focus(False)
 
     @on(Input.Submitted, '#queue-input')
     def queue_submitted(self, event: Input.Submitted) -> None:
+        self.state.update_conf('queue_name', event.value)
         next = self.get_widget_by_id('mqueue-input')
         assert isinstance(next, Input)
         if next.value == '':
             next.value = event.input.value
+            self.state.update_conf('multi_node_queue_name', event.value)
         next.focus(False)
 
     @on(Input.Submitted, '#mqueue-input')
-    def mqueue_submitted(self) -> None:
+    def mqueue_submitted(self, event: Input.Submitted) -> None:
+        self.state.update_conf('multi_node_queue_name', event.value)
         self.app._focus_next()  # type: ignore
 
     @on(Button.Pressed, '#btn-edit-attrs')
@@ -405,12 +426,13 @@ async def _run_test_job(self, jd: TestJobsDialog, job_no: int, label: str,
             return False
 
     def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job:
-        s = self.state.scheduler
-        assert s is not None
+        scheduler = self._get_scheduler()
+        assert scheduler is not None
 
-        ex = JobExecutor.get_instance(s)
+        ex = JobExecutor.get_instance(scheduler)
 
         attrs = JobAttributes()
+
         account = self.state.conf.get('account', '')
         if rspec is not None and rspec.computed_node_count > 1:
             queue = self.state.conf.get('multi_node_queue_name', '')
@@ -423,7 +445,7 @@ def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job:
         for attr in self.state.attrs:
             if re.match(attr.filter, test_name):
                 attrs.set_custom_attribute(attr.name, attr.value)
-
+        log.write(f'attrs: {attrs}, rspec: {rspec}\n')
         job = Job(JobSpec('/bin/date', attributes=attrs, resources=rspec))
         ex.submit(job)
         log.write('Job submitted\n')

tests/installer/panels/intro_panel.py

@@ -41,4 +41,5 @@ async def activate(self) -> None:
             if result == 'quit':
                 self.app.exit()
             if result == 'update':
+                self.state.backup_conf()
                 self.app.disable_install()  # type: ignore

tests/installer/state.py

@@ -3,6 +3,7 @@
 import json
 import os
 import pathlib
+import shutil
 import types
 
 import requests
@@ -19,6 +20,10 @@
 Attr = namedtuple('Attr', ['filter', 'name', 'value'])
 
 
+_EXECUTOR_LABELS = {'slurm': 'Slurm', 'pbs': 'PBS', 'cobalt': 'Cobalt', 'lsf': 'LSF',
+                    'none': 'None'}
+
+
 async def _to_thread(func, /, *args, **kwargs):  # type: ignore
     loop = asyncio.get_running_loop()
     func_call = functools.partial(func, *args, **kwargs)
@@ -43,10 +48,33 @@ def getoption(self, name: str) -> Union[str, int, bool, None]:
         return self.dict[name]
 
 
+class Exec:
+    def __init__(self, pair: str) -> None:
+        array = pair.split(':')
+        assert len(array) > 0 and len(array) <= 3
+        self.name = array[0]
+        self.launcher = ''
+        self.url = ''
+        if len(array) > 1:
+            self.launcher = array[1]
+        if len(array) > 2:
+            self.url = array[2]
+
+    def __str__(self) -> str:
+        if self.launcher == '':
+            return self.name
+        else:
+            return f'{self.name}:{self.launcher}'
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
 class State:
     def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType):
         self.conf = ci_runner.read_conf('testing.conf')
         self.env = conftest._discover_environment(ConfWrapper(self.conf))
+        self.translate_launcher = conftest._translate_launcher
         log.write('Conf: ' + str(self.conf) + '\n')
         log.write(str(self.env) + '\n')
         self.disable_install = False
@@ -56,6 +84,9 @@ def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType):
         self.attrs = self._parse_attributes()
         self.run_test_job = True
         self.has_key = KEY_PATH.exists()
+        self.conf_backed_up = False
+        self.execs, self.batch_exec = self._parse_executors()
+        log.write(f'execs: {self.execs}, bexec: {self.batch_exec}\n')
         if self.has_key:
             with open(KEY_PATH, 'r') as f:
                 self.key = f.read().strip()
@@ -77,6 +108,7 @@ def _parse_attributes(self) -> List[Attr]:
     def _write_conf_value(self, name: str, value: str) -> None:
         self.conf[name] = value
         nlen = len(name)
+        found = False
         with open('testing.conf', 'r') as old:
             with open('testing.conf.new', 'w') as new:
                 for line in old:
@@ -85,8 +117,11 @@ def _write_conf_value(self, name: str, value: str) -> None:
                         new.write(line)
                     elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=']:
                         new.write(f'{name} = {value}\n')
+                        found = True
                     else:
                         new.write(line)
+                if not found:
+                    new.write(line)
         os.rename('testing.conf.new', 'testing.conf')
 
     def update_conf(self, name: str, value: str) -> None:
@@ -95,6 +130,10 @@ def update_conf(self, name: str, value: str) -> None:
         else:
             self._write_conf_value(name, value)
 
+    def backup_conf(self) -> None:
+        shutil.copyfile('testing.conf', 'testing.conf.bk')
+        self.conf_backed_up = True
+
     async def request(self, query: str, data: Dict[str, object], title: str,
                       error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \
             -> Optional[Dict[str, object]]:
@@ -166,14 +205,63 @@ def _write_custom_attrs(self, attrs: List[Attr]) -> None:
                     new.write(line)
         os.rename('testing.conf.new', 'testing.conf')
 
-    def get_executor(self) -> Tuple[str, str]:
+    def _executor_label(self, name: Optional[str]) -> Optional[str]:
+        if name in _EXECUTOR_LABELS:
+            return _EXECUTOR_LABELS[name]
+        else:
+            return name
+
+    def get_auto_executor(self) -> Optional[str]:
         if self.env['has_slurm']:
-            return ('Slurm', 'slurm')
+            return 'slurm'
         if self.env['has_pbs']:
-            return ('PBS', 'pbs')
+            return 'pbs'
         if self.env['has_lsf']:
-            return ('LSF', 'lsf')
+            return 'lsf'
         if self.env['has_cobalt']:
-            return ('Cobalt', 'cobalt')
+            return 'cobalt'
+
+        return 'none'
+
+    def get_batch_executor(self) -> Tuple[Optional[str], Optional[str]]:
+        return self._executor_label(self.batch_exec), self.batch_exec
+
+    def _parse_executors(self) -> Tuple[List[Exec], Optional[str]]:
+        conf_exec = self.conf.get('executors', '')
+        execs = []
+        for exec in conf_exec.split(','):
+            exec = exec.strip()
+            if exec != '':
+                execs.append(Exec(exec))
+        if len(execs) == 0:
+            execs = [Exec('auto')]
+        batch = None
+        for exec in execs:
+            if exec.name == 'auto' or exec.name == 'auto_q':
+                batch = self.get_auto_executor()
+            elif exec.name not in ['local', 'batch-test']:
+                batch = exec.name
+
+        return execs, batch
+
+    def _executors_str(self) -> str:
+        return ', '.join([str(x) for x in self.execs])
+
+    def set_batch_executor(self, scheduler: str) -> None:
+        auto_exec = self.get_auto_executor()
+        if auto_exec == scheduler or scheduler == 'none':
+            self.execs = [Exec('auto')]
+        elif scheduler == 'local' and auto_exec == 'none':
+            self.execs = [Exec('auto')]
+        else:
+            existing = None
+            for exec in self.execs:
+                if exec.name not in ['auto', 'auto_q', 'local', 'batch-test']:
+                    exec.name = scheduler
+                    exec.launcher = 'auto_l'
+            if not existing:
+                self.execs.append(Exec(scheduler))
+
+        log.write(f'set_be({scheduler}): execs: {self.execs}, str: {self._executors_str()}\n')
 
-        return ('None', 'none')
+        self._write_conf_value('executors', self._executors_str())