Merge pull request #1654 from vkarak/bugfix/lmod-crash-pre-82

[bugfix] Fix module backend implementation for Lmod versions < 8.2

Author: Vasileios Karakasis

.github/workflows/main.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        modules-version: [Lmod, Tmod32, Tmod4]
+        modules-version: [Lmod, Lmod77, Tmod32, Tmod4]
     steps:
     - uses: actions/checkout@v2
     - name: Build Image for ${{ matrix.modules-version }}

ci-scripts/dockerfiles/Lmod77.dockerfile

@@ -0,0 +1,52 @@
+#
+# Execute this from the top-level ReFrame source directory
+#
+
+#
+# LMod versions prior to 8.2 emitted Python commands differently, so we use this
+# Dockerfile to test the bindings of older versions
+#
+
+
+FROM ubuntu:20.04
+
+ENV TZ=Europe/Zurich
+ENV DEBIAN_FRONTEND=noninteractive
+ENV _LMOD_VER=7.7
+
+# ReFrame user
+RUN useradd -ms /bin/bash rfmuser
+
+# ReFrame requirements
+RUN \
+  apt-get -y update && \
+  apt-get -y install ca-certificates && \
+  update-ca-certificates && \
+  apt-get -y install gcc && \
+  apt-get -y install make && \
+  apt-get -y install git && \
+  apt-get -y install python3 python3-pip
+
+# Required utilities
+RUN apt-get -y install wget
+
+# Install Lmod
+RUN \
+  apt-get -y install lua5.3 lua-bit32:amd64 lua-posix:amd64 lua-posix-dev liblua5.3-0:amd64 liblua5.3-dev:amd64 tcl tcl-dev tcl8.6 tcl8.6-dev:amd64 libtcl8.6:amd64 lua-filesystem:amd64 lua-filesystem-dev:amd64 && \
+  wget -q https://github.com/TACC/Lmod/archive/${_LMOD_VER}.tar.gz -O lmod.tar.gz && \
+  tar xzf lmod.tar.gz && \
+  cd Lmod-${_LMOD_VER} && \
+  ./configure && make install
+
+ENV BASH_ENV=/usr/local/lmod/lmod/init/profile
+
+USER rfmuser
+
+# Install ReFrame from the current directory
+COPY --chown=rfmuser . /home/rfmuser/reframe/
+
+WORKDIR /home/rfmuser/reframe
+
+RUN ./bootstrap.sh
+
+CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py -v"]

reframe/core/modules.py

@@ -492,6 +492,21 @@ def emit_load_instr(self, module):
     def emit_unload_instr(self, module):
         '''Emit the instruction that unloads module.'''
 
+    def process(self, source):
+        '''Process the Python source emitted by the Python bindings of the
+        different backends.
+
+        Backends should call this before executing any Python commands.
+
+        :arg source: The Python source code to be executed.
+        :returns: The modified Python source code to be executed. By default
+            ``source`` is returned unchanged.
+
+        .. versionadded:: 3.4
+
+        '''
+        return source
+
     def __repr__(self):
         return type(self).__name__ + '()'
 
@@ -562,7 +577,7 @@ def _execute(self, cmd, *args):
                                       completed.stderr,
                                       completed.returncode)
 
-        exec(completed.stdout)
+        exec(self.process(completed.stdout))
         return completed.stderr
 
     def available_modules(self, substr):
@@ -692,7 +707,7 @@ def _execute(self, cmd, *args):
         with open(exec_match.group(1), 'r') as content_file:
             cmd = content_file.read()
 
-        exec(cmd)
+        exec(self.process(cmd))
         return completed.stderr
 
 
@@ -740,7 +755,7 @@ def _execute(self, cmd, *args):
         modulecmd = self.modulecmd(cmd, *args)
         completed = osext.run_command(modulecmd, check=False)
         namespace = {}
-        exec(completed.stdout, {}, namespace)
+        exec(self.process(completed.stdout), {}, namespace)
 
         # _mlstatus is set by the TMod4 only if the command was unsuccessful,
         # but Lmod sets it always
@@ -824,6 +839,17 @@ def __init__(self):
     def name(self):
         return 'lmod'
 
+    def process(self, source):
+        major, minor, *_ = self.version().split('.')
+        major, minor = int(major), int(minor)
+        if (major, minor) < (8, 2):
+            # Older Lmod versions do not emit an `import os` and emit an
+            # invalid `false` statement in case of errors; we fix these here
+            return 'import os\n\n' + source.replace('false',
+                                                    '_mlstatus = False')
+
+        return source
+
     def modulecmd(self, *args):
         return ' '.join([self._lmod_cmd, 'python', *args])