Merge pull request #2140 from ekouts/bugfix/autodetect_modules

[bugfix] Use partition's and system's environment when autodetecting the topology

Author: Vasileios Karakasis

reframe/core/schedulers/slurm.py

@@ -292,7 +292,7 @@ def filternodes(self, job, nodes):
         # Collect options that restrict node selection, but we need to first
         # create a mutable list out of the immutable SequenceView that
         # sched_access is
-        options = list(job.sched_access + job.options + job.cli_options)
+        options = job.sched_access + job.options + job.cli_options
         option_parser = ArgumentParser()
         option_parser.add_argument('--reservation')
         option_parser.add_argument('-p', '--partition')

reframe/frontend/autodetect.py

@@ -10,10 +10,10 @@
 import tempfile
 
 import reframe as rfm
+import reframe.core.runtime as runtime
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ConfigError
 from reframe.core.logging import getlogger
-from reframe.core.runtime import runtime
 from reframe.core.schedulers import Job
 from reframe.utility.cpuinfo import cpuinfo
 
@@ -60,7 +60,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 def _subschema(fragment):
     '''Create a configuration subschema.'''
 
-    full_schema = runtime().site_config.schema
+    full_schema = runtime.runtime().site_config.schema
     return {
         '$schema': full_schema['$schema'],
         'defs': full_schema['defs'],
@@ -111,29 +111,28 @@ def _is_part_local(part):
 
 
 def _remote_detect(part):
-    def _emit_script(job):
+    def _emit_script(job, env):
         launcher_cmd = job.launcher.run_command(job)
         commands = [
             f'./bootstrap.sh',
             f'{launcher_cmd} ./bin/reframe --detect-host-topology=topo.json'
         ]
-        job.prepare(commands, trap_errors=True)
+        job.prepare(commands, env, trap_errors=True)
 
     getlogger().info(
         f'Detecting topology of remote partition {part.fullname!r}: '
         f'this may take some time...'
-
     )
     topo_info = {}
     try:
-        prefix = runtime().get_option('general/0/remote_workdir')
+        prefix = runtime.runtime().get_option('general/0/remote_workdir')
         with _copy_reframe(prefix) as dirname:
             with osext.change_dir(dirname):
                 job = Job.create(part.scheduler,
                                  part.launcher_type(),
                                  name='rfm-detect-job',
                                  sched_access=part.access)
-                _emit_script(job)
+                _emit_script(job, [part.local_env])
                 getlogger().debug('submitting detection script')
                 _log_contents(job.script_filename)
                 job.submit()
@@ -149,7 +148,7 @@ def _emit_script(job):
 
 
 def detect_topology():
-    rt = runtime()
+    rt = runtime.runtime()
     detect_remote_systems = rt.get_option('general/0/remote_detect')
     topo_prefix = os.path.join(os.getenv('HOME'), '.reframe/topology')
     for part in rt.system.partitions:
@@ -213,12 +212,22 @@ def detect_topology():
         if not found_procinfo:
             # No topology found, try to auto-detect it
             getlogger().debug(f'> no topology file found; auto-detecting...')
+            modules = list(rt.system.preload_environ.modules)
+            vars = dict(rt.system.preload_environ.variables.items())
             if _is_part_local(part):
+                modules += part.local_env.modules
+                vars.update(part.local_env.variables)
+
                 # Unconditionally detect the system for fully local partitions
-                part.processor._info = cpuinfo()
+                with runtime.temp_environment(modules=modules, variables=vars):
+                    part.processor._info = cpuinfo()
+
                 _save_info(topo_file, part.processor.info)
             elif detect_remote_systems:
-                part.processor._info = _remote_detect(part)
+                with runtime.temp_environment(modules=temp_modules,
+                                              variables=temp_vars):
+                    part.processor._info = _remote_detect(part)
+
                 if part.processor.info:
                     _save_info(topo_file, part.processor.info)
 

reframe/utility/__init__.py

@@ -1366,6 +1366,12 @@ class SequenceView(collections.abc.Sequence):
     :raises TypeError: If the container does not fulfill the
         :py:class:`collections.abc.Sequence` interface.
 
+    .. note::
+
+       You can concatenate a :class:`SequenceView` with a container of the
+       same type as the underlying container of the view, in which case a new
+       container with the concatenated elements will be returned.
+
     '''
 
     def __init__(self, container):
@@ -1416,13 +1422,16 @@ def __reversed__(self):
         return self.__container.__reversed__()
 
     def __add__(self, other):
-        if not isinstance(other, collections.abc.Sequence):
+        if not isinstance(other, type(self.__container)):
             return NotImplemented
 
-        return SequenceView(self.__container + other)
+        return self.__container + other
+
+    def __radd__(self, other):
+        if not isinstance(other, type(self.__container)):
+            return NotImplemented
 
-    def __iadd__(self, other):
-        return NotImplemented
+        return other + self.__container
 
     def __eq__(self, other):
         if isinstance(other, SequenceView):

unittests/test_utility.py

@@ -1072,14 +1072,18 @@ def test_sequence_view():
     # Assert immutability
     m = l + [3, 4]
     assert [1, 2, 2, 3, 4] == m
-    assert isinstance(m, util.SequenceView)
+    assert isinstance(m, list)
 
-    m = l
-    l += [3, 4]
-    assert m is not l
-    assert [1, 2, 2] == m
-    assert [1, 2, 2, 3, 4] == l
-    assert isinstance(l, util.SequenceView)
+    m_orig = m = util.SequenceView([1])
+    m += [3, 4]
+    assert m is not m_orig
+    assert [1] == m_orig
+    assert [1, 3, 4] == m
+    assert isinstance(m, list)
+
+    n = m + l
+    assert [1, 3, 4, 1, 2, 2] == n
+    assert isinstance(n, list)
 
     with pytest.raises(TypeError):
         l[1] = 3
@@ -1542,6 +1546,7 @@ def test_jsonext_dumps():
     )
     assert '{"(1, 2, 3)": 1}' == jsonext.dumps({(1, 2, 3): 1})
 
+
 # Classes to test JSON deserialization
 
 class _D(jsonext.JSONSerializable):