Merge pull request #388 from xdelaruelle/improve_modules

vsoch · web-flow · commit cf4e30221ead · 2021-06-06T14:42:37.000-06:00
Improve tcl/lua module code &amp; test
diff --git a/.github/workflows/test-container.yml b/.github/workflows/test-container.yml
@@ -13,7 +13,7 @@ jobs:
         run: |
           sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
 
-      - name: Install LMOD
+      - name: Install Lmod
         run: |
           PKG_VERSION=8.4.27
           PKG_URL="https://github.com/TACC/Lmod/archive/${PKG_VERSION}.tar.gz"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -15,9 +15,9 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+          sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev tcsh
 
-      - name: Install environment modules
+      - name: Install Environment Modules
         if: ${{ matrix.module == 'tcl' }}
         run: |
          curl -LJO https://github.com/cea-hpc/modules/releases/download/v4.7.1/modules-4.7.1.tar.gz
@@ -28,7 +28,7 @@ jobs:
          make
          sudo make install
 
-      - name: Install LMOD
+      - name: Install Lmod
         if: ${{ matrix.module == 'lmod' }}
         run: |
           PKG_VERSION=8.4.27
@@ -54,16 +54,27 @@ jobs:
           source activate shpc
           pip install -e .
 
-      - name: Run python module tests
-        if: ${{ matrix.module == 'lmod' }}
+      - name: Run python module tests (bash)
+        shell: bash
         run: |
+          case "${{ matrix.module }}" in
+            lmod)
+              . /usr/share/lmod/lmod/init/bash
+              ;;
+            tcl)
+              . /usr/local/Modules/init/bash
+              shopt expand_aliases || true
+              shopt -s expand_aliases
+              ;;
+          esac
+
           export PATH="/usr/share/miniconda/bin:$PATH"
-          . /usr/share/lmod/lmod/init/sh
           source activate shpc
           shpc config set container_tech:${{ matrix.container_tech }}
           shpc config set module_sys:${{ matrix.module }}
           shpc config set enable_tty:false
           shpc install python:3.9.5-alpine
+
           module use ./modules
           module load python/3.9.5-alpine
           alias
@@ -76,24 +87,35 @@ jobs:
 
           printf "\n\nmodule help ============================================\n"
           module help python/3.9.5-alpine
-          python-exec echo donuts
-          python-run python --version
 
-      - name: Run python module tests
-        if: ${{ matrix.module == 'tcl' }}
-        shell: bash
+          python-exec echo donuts >test_output
+          cat test_output
+          grep --quiet donuts test_output
+
+          python-run python --version >test_output
+          cat test_output
+          grep --quiet 'Python 3.9.5' test_output
+          rm test_output
+
+      - name: Run python module tests (tcsh)
+        shell: tcsh -e {0}
         run: |
-          export PATH="/usr/share/miniconda/bin:$PATH"
-          . /usr/local/Modules/init/sh
-          source activate shpc
+          switch ("${{ matrix.module }}")
+            case lmod:
+              source /usr/share/lmod/lmod/init/tcsh
+              breaksw
+            case tcl:
+              source /usr/local/Modules/init/tcsh
+              breaksw
+          endsw
+
+          source /usr/share/miniconda/etc/profile.d/conda.csh
+          conda activate shpc
           shpc config set container_tech:${{ matrix.container_tech }}
           shpc config set module_sys:${{ matrix.module }}
           shpc config set enable_tty:false
           shpc install python:3.9.5-alpine
 
-          shopt expand_aliases || true
-          shopt -s expand_aliases
-
           module use ./modules
           module load python/3.9.5-alpine
           alias
@@ -106,5 +128,12 @@ jobs:
 
           printf "\n\nmodule help ============================================\n"
           module help python/3.9.5-alpine
-          python-exec echo donuts
-          python-run python --version
+
+          python-exec echo donuts >test_output
+          cat test_output
+          grep --quiet donuts test_output
+
+          python-run python --version >test_output
+          cat test_output
+          grep --quiet 'Python 3.9.5' test_output
+          rm test_output
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 Singularity HPC is optimized for managing containers in an HPC environment. Currently, this includes
 module technologies:
 
- - [LMOD](https://lmod.readthedocs.io/en/latest/)
+ - [Lmod](https://lmod.readthedocs.io/en/latest/)
  - [Environment Modules](http://modules.sourceforge.net/)
 
 And container technologies:
diff --git a/docs/getting_started/developer-guide.rst b/docs/getting_started/developer-guide.rst
@@ -309,8 +309,8 @@ use is the following:
 Development or Testing
 ======================
 
-If you first want to test singularity-hpc (shpc) with an LMOD installed in 
-a container, a ``Dockerfile`` is provided for LMOD, and ``Dockerfile.tcl``
+If you first want to test singularity-hpc (shpc) with an Lmod installed in 
+a container, a ``Dockerfile`` is provided for Lmod, and ``Dockerfile.tcl``
 for tcl modules. The assumption is that
 you have a module system installed on your cluster or in the container. If not, you
 can find instructions `here for lmod <https://lmod.readthedocs.io/en/latest/030_installing.html>`_
diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst
@@ -5,7 +5,7 @@ Getting Started
 ===============
 
 Singularity Registry (HPC) is a tool that makes it easy to install containers as
-LMOD modules. You can create your own registry entries (e.g., a specification
+Lmod modules. You can create your own registry entries (e.g., a specification
 to pull a particular container and expose some number of entrypoints) or
 the library also provides you with a community set.
 
diff --git a/docs/getting_started/installation.rst b/docs/getting_started/installation.rst
@@ -52,7 +52,7 @@ $ which shpc
 
 This executable should be accessible by an administrator, or anyone that you want
 to be able to manage containers. Your user base will be interacting with your
-containers via LMOD, so they do not need access to `shpc`. 
+containers via Lmod, so they do not need access to `shpc`. 
 If you are a user creating your own folder of modules, you can add them
 to your module path.
 
@@ -94,7 +94,7 @@ and interacting with your modules.
 Environment Modules
 -------------------
 
-If you are using `environment modules (tcl) <http://modules.sourceforge.net/>`_
+If you are using `Environment Modules (tcl) <http://modules.sourceforge.net/>`_
 and you find that your aliases do not expand, you can use `shopt <https://www.gnu.org/software/bash/manual/html_node/The-Shopt-Builtin.html>`_ to fix this issue:
 
 .. code-block:: console
diff --git a/docs/getting_started/use-cases.rst b/docs/getting_started/use-cases.rst
@@ -42,7 +42,7 @@ Make sure that lmod knows about the folder
 
     $ module use /opt/lmod/shpc
      
-(And likely if you administer an LMOD install you have your preferred way of
+(And likely if you administer an Lmod install you have your preferred way of
 doing this). And then you can use your modules just as you would that are provided on
 your cluster.
 
diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst
@@ -59,7 +59,7 @@ then install a container:
     $ shpc install python
     
 Add the modules folder to your lmod (you can run this in a bash profile or
-manually, and note that if you want to use environment modules, you need to add
+manually, and note that if you want to use Environment Modules, you need to add
 ``--module-sys tcl``).
 
 
@@ -336,7 +336,7 @@ commands. Aliases that are custom to the container are not modified.
 Module Software
 ===============
 
-The default module software is currently LMOD, and there is also support for environment
+The default module software is currently Lmod, and there is also support for environment
 modules that only use tcl (tcl). If you
 are interested in adding another module type, please `open an issue <https://github.com/singularityhub/singularity-hpc>`_ and
 provide description and links to what you have in mind. You can either specify the
@@ -513,7 +513,7 @@ You can also install a specific tag (as shown in list).
     $ shpc install python:3.9.2-alpine
     
 
-Note that LMOD is the default for the module system, and Singularity for
+Note that Lmod is the default for the module system, and Singularity for
 the container technology.
 If you don't have any module software on your system, you can now test interacting
 with the module via the :ref:`getting_started-development` instructions.
@@ -676,7 +676,7 @@ Test
 ----
 
 Singularity HPC makes it easy to test the full flow of installing and interacting
-with modules. This functionality requires a module system (e.g., LMOD) to be installed,
+with modules. This functionality requires a module system (e.g., Lmod) to be installed,
 and the assumption is that the test is being run in a shell environment where any
 supporting modules (e.g., loading Singularity or Podman) would be found if needed.
 This is done by way of extending the exported ``$MODULEPATH``. To run a test, you
diff --git a/docs/index.rst b/docs/index.rst
@@ -13,7 +13,7 @@ Singularity Registry (HPC)
 Singularity Registry HPC (shpc) allows you to install containers as modules.
 Currently, this includes:
 
- - `LMOD <https://lmod.readthedocs.io/en/latest/>`_
+ - `Lmod <https://lmod.readthedocs.io/en/latest/>`_
  - `Environment Modules <http://modules.sourceforge.net/>`_
 
 And container technologies:
diff --git a/shpc/main/__init__.py b/shpc/main/__init__.py
@@ -11,7 +11,7 @@
 
 def get_client(quiet=False, **kwargs):
     """
-    Get a singularity HPC client based on the backend (e.g., LMOD)
+    Get a singularity HPC client based on the backend (e.g., Lmod)
     and container technology (currently just Singularity) of interest.
 
     Parameters
diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua
@@ -1,4 +1,4 @@
--- LMOD Module
+-- Lmod Module
 -- Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
 -- ##
 -- {{ name }} on {{ creation_date }}
@@ -58,7 +58,7 @@ conflict(myModuleName(){% if aliases %}{% for alias in aliases %},"{{ alias.name
 
 -- exec functions to provide "alias" to module commands
 {% if aliases %}{% for alias in aliases %}
-set_shell_function("{{ alias.name }}", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $@", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $*")
+set_shell_function("{{ alias.name }}", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $@", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}")
 {% endfor %}{% endif %}
 
 {% if aliases %}
@@ -68,10 +68,10 @@ if (myShellName() == "bash") then
 end{% endif %}
 
 -- A customizable exec function
-set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " $@",  execCmd .. " --entrypoint \"\"" .. containerPath .. " $*")
+set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " $@",  execCmd .. " --entrypoint \"\" " .. containerPath)
 
 -- Always provide a container run
-set_shell_function("{|module_name|}-run", runCmd .. " $@",  runCmd .. " $*")
+set_shell_function("{|module_name|}-run", runCmd .. " $@",  runCmd)
 
 -- Inspect runscript or deffile easily!
 set_shell_function("{|module_name|}-inspect", inspectCmd,  inspectCmd)
diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl
@@ -1,4 +1,4 @@
-#%Module 1.0
+#%Module
 
 #=====
 # Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
@@ -32,6 +32,10 @@ proc ModulesHelp { } {
 
 }
 
+# Environment
+setenv DOCKER_OPTS ""
+setenv DOCKER_COMMAND_OPTS ""
+
 # Variables
 
 set name        "{{ name }}"
@@ -60,21 +64,21 @@ set runCmd "{{ command }} \${DOCKER_OPTS} run -i{% if tty %}t{% endif %} \${DOCK
 set inspectCmd "{{ command }} \${DOCKER_OPTS} inspect ${containerPath}" 
 
 # set_shell_function takes bashStr and cshStr
-set-alias "{|module_name|}-shell" "${shellCmd}"
+set-alias {|module_name|}-shell "${shellCmd}"
 
 # exec functions to provide "alias" to module commands
 {% if aliases %}{% for alias in aliases %}
-set-alias "{{ alias.name }}" "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.command }} $*"
+set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.command }}"
 {% endfor %}{% endif %}
 
 # A customizable exec function
-set-alias "{|module_name|}-exec" "${execCmd} --entrypoint \"\" ${containerPath} $*"
+set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath}"
 
 # Always provide a container run
-set-alias "{|module_name|}-run" "${runCmd} $*"
+set-alias {|module_name|}-run "${runCmd}"
 
 # Inspect runscript or deffile easily!
-set-alias "{|module_name|}-inspect" "${inspectCmd} ${containerPath} $*"
+set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"
 
 #=====
 # Module options
diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua
@@ -1,4 +1,4 @@
--- LMOD Module
+-- Lmod Module
 -- Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
 -- ##
 -- {{ name }} on {{ creation_date }}
@@ -51,7 +51,7 @@ setenv ("SINGULARITY_COMMAND_OPTS", "")
 local containerPath = '{{ container_sif }}'
 local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " .. containerPath
 local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} "
-local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " .. containerPath .. " ${SINGULARITY_COMMAND_ARGS}"
+local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " .. containerPath
 local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " 
 
 -- set_shell_function takes bashStr and cshStr
@@ -62,7 +62,7 @@ conflict(myModuleName(){% if aliases %}{% for alias in aliases %},"{{ alias.name
 
 -- exec functions to provide "alias" to module commands
 {% if aliases %}{% for alias in aliases %}
-set_shell_function("{{ alias.name }}", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} containerPath .. " {{ alias.command }} $@", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} containerPath .. " {{ alias.command }} $*")
+set_shell_function("{{ alias.name }}", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} containerPath .. " {{ alias.command }} $@", execCmd .. {% if alias.options %} "{{ alias.options }} " .. {% endif %} containerPath .. " {{ alias.command }}")
 {% endfor %}{% endif %}
 
 {% if aliases %}
@@ -72,10 +72,10 @@ if (myShellName() == "bash") then
 end{% endif %}
 
 -- A customizable exec function
-set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " ${SINGULARITY_COMMAND_ARGS}  $@",  execCmd .. containerPath .. " ${SINGULARITY_COMMAND_ARGS} $*")
+set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " $@",  execCmd .. containerPath)
 
 -- Always provide a container run
-set_shell_function("{|module_name|}-run", runCmd .. " $@",  runCmd .. " $*")
+set_shell_function("{|module_name|}-run", runCmd .. " $@",  runCmd)
 
 -- Inspect runscript or deffile easily!
 set_shell_function("{|module_name|}-inspect-runscript", inspectCmd .. " -r  " .. containerPath,  inspectCmd .. containerPath)
diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl
@@ -1,4 +1,4 @@
-#%Module 1.0
+#%Module
 
 #=====
 # Created by singularity-hpc (https://github.com/singularityhub/singularity-hpc)
@@ -40,8 +40,8 @@ proc ModulesHelp { } {
 }
 
 # Environment
-set ::env(SINGULARITY_OPTS) ""
-set ::env(SINGULARITY_COMMAND_OPTS) ""
+setenv SINGULARITY_OPTS ""
+setenv SINGULARITY_COMMAND_OPTS ""
 
 # Variables
 
@@ -68,27 +68,27 @@ setenv SINGULARITY_SHELL {{ singularity_shell }}
 # interactive shell to any container, plus exec for aliases
 set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} ${containerPath}" 
 set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} "
-set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} ${containerPath} \${SINGULARITY_COMMAND_ARGS}"
+set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} ${containerPath}"
 set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} " 
 
 # set_shell_function takes bashStr and cshStr
-set-alias "{|module_name|}-shell" "${shellCmd} $*"
+set-alias {|module_name|}-shell "${shellCmd}"
 
 
 # exec functions to provide "alias" to module commands
 {% if aliases %}{% for alias in aliases %}
-set-alias "{{ alias.name }}" "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} ${containerPath} {{ alias.command }} $*"
+set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} ${containerPath} {{ alias.command }}"
 {% endfor %}{% endif %}
 
 # A customizable exec function
-set-alias "{|module_name|}-exec" "${execCmd} ${containerPath} \${SINGULARITY_COMMAND_ARGS} $*"
+set-alias {|module_name|}-exec "${execCmd} ${containerPath}"
 
 # Always provide a container run
-set-alias "{|module_name|}-run" "${runCmd} $*"
+set-alias {|module_name|}-run "${runCmd}"
 
 # Inspect runscript or deffile easily!
-set-alias "{|module_name|}-inspect-runscript" "${inspectCmd} -r ${containerPath}"
-set-alias "{|module_name|}-inspect-deffile" "${inspectCmd} -d ${containerPath}"
+set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}"
+set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"
 
 #=====
 # Module options
diff --git a/shpc/main/modules/templates/test.sh b/shpc/main/modules/templates/test.sh
diff --git a/shpc/settings.yml b/shpc/settings.yml
