Implemented "default_version==false" for TCL (#517)

* Implemented "default_version" for TCL
* refactor to use shared logic for default version template loading
I am also adding tests for both tcl and lmod default versions
* correct default version cases
* refactor to allow default_version to have multiple options
* change order of defaults for default_version
* proposal for handling default version on uninstall

instead of parsing each case, we can simply pretend that the last module installed
(determined by the creation or modify date) was just installed, and for all variants
of default_version this will work as it will either write this version as latest,
or just keep the dummy version. The exception is for first installed, for which we need
to pass a boolean that says "force calling this the first install." of course linux
does not do creation date well, so we do our best with modify date instead. Note I
have not tested this but am pushing the commit to further discuss!

* Clarified the logic in write_version_file and update_version_file
- New _set_default_version method to set the default version
- write_version_file and update_version_file now decide which is
  the first/last installed version (no more _default_version_first_installed)
* Expanded _cleanup to delete directories if they only contain ".version"
* TCL is now supported
* Merged update_version_file into write_version_file
* write_version_file can now detect the first/last version on its own
* Renamed the config file value to "module_sys" to match the other option
* Made can_be_deleted more generic, by taking the list of files to ignore as a parameter
* Reintroduced the tag parameter

Made it explicit that the tag write_version_file gets represents the version being installed

* Renamed _sys_module_default_version to _module_sys_default_version for consistency
* We can set default as None since it's not required.
* Use `latest_tag_installed` if possible

Co-authored-by: Matthieu Muffato <[email protected]>
Co-authored-by: vsoch <[email protected]>

Author: muffato

docs/conf.py

@@ -290,4 +290,4 @@
 
 
 def setup(app):
-    app.add_stylesheet("sphinx-argparse.css")
+    app.add_css_file("sphinx-argparse.css")

docs/getting_started/developer-guide.rst

@@ -257,7 +257,7 @@ For these global scripts, the user can select to use it in their settings.yaml.
 We will eventually write a command to list global wrappers available, so if you add a new one future users will know
 about it. For alias wrapper scripts, the following variables are passed for rendering:
 
-.. list-table:: Title
+.. list-table:: Wrapper Script Variables
    :widths: 15 15 40 30
    :header-rows: 1
 
@@ -306,7 +306,7 @@ If you want to write a custom container.yaml script:
 
 The following variables are passed for rendering.
 
-.. list-table:: Title
+.. list-table:: Container YAML Alias Variables
    :widths: 15 15 40 30
    :header-rows: 1
 
@@ -523,7 +523,7 @@ Registry Yaml Fields
 
 Fields include:
 
-.. list-table:: Title
+.. list-table:: Registry YAML Fields
    :widths: 25 65 10
    :header-rows: 1
 
@@ -571,7 +571,7 @@ A complete table of features is shown here. The
 
 Fields include:
 
-.. list-table:: Title
+.. list-table:: Features
    :widths: 20 20 20 10 10 10
    :header-rows: 1
 

docs/getting_started/user-guide.rst

@@ -178,8 +178,8 @@ variable replacement. A summary table of variables is included below, and then f
      - a timestamp to keep track of when you last saved
      - never
    * - default_version
-     - A boolean to indicate generating a .version file (LMOD or lua modules only)
-     - true
+     - Should a default version be used?
+     - module_sys
    * - singularity_module
      - if defined, add to module script to load this Singularity module first
      - null
@@ -360,6 +360,18 @@ you can add or remove entries via the config variable ``registry``
 and "set" is used to set a key value pair.
 
 
+Default Version
+---------------
+
+The default version setting is there to support you telling shpc how you want module versions to be selected.
+There are four options:
+
+ - ``null`` do not set any kind of default version, it will be manually controlled by the installer (``false`` also supported for backwards compatibility)
+ - ``module_sys``: allow the module software to choose (``true`` also supported for backwards compatibility)
+ - ``last_installed``: always set default version to the last version installed
+ - ``first_installed``: only set default version for the first installed
+
+
 Module Names
 ------------
 

shpc/main/modules/base.py

@@ -12,7 +12,6 @@
 
 from datetime import datetime
 import os
-from pathlib import Path
 import shutil
 import subprocess
 import sys
@@ -23,6 +22,8 @@
 
 class ModuleBase(BaseClient):
     def __init__(self, **kwargs):
+
+        # Files for module software to generate depending on user setting
         super(ModuleBase, self).__init__(**kwargs)
         self.here = os.path.dirname(inspect.getfile(self.__class__))
 
@@ -68,7 +69,7 @@ def _cleanup(self, module_dir):
 
         # If directories above it are empty, remove
         while module_dir != self.settings.module_base:
-            if not os.path.exists(module_dir) or os.listdir(module_dir):
+            if not utils.can_be_deleted(module_dir, [".version"]):
                 break
             shutil.rmtree(module_dir)
             module_dir = os.path.dirname(module_dir)
@@ -125,6 +126,13 @@ def _uninstall(self, module_dir, name, force=False):
         else:
             logger.warning("%s does not exist." % name)
 
+        # parent of versioned directory has module .version
+        module_dir = os.path.dirname(module_dir)
+
+        # update the default version file, if other versions still present
+        if os.path.exists(module_dir):
+            self.write_version_file(module_dir)
+
     def _test_setup(self, tmpdir):
         """
         Setup tests, including changes to settings or test directory
@@ -353,12 +361,8 @@ def install(self, name, tag=None, **kwargs):
                 shutil.copyfile(container_path, container_dest)
             container_path = container_dest
 
-        # Add a .version file to indicate the level of versioning (not for tcl)
-        if self.module_extension != "tcl" and self.settings.default_version == True:
-            version_dir = os.path.join(self.settings.module_base, uri)
-            version_file = os.path.join(version_dir, ".version")
-            if not os.path.exists(version_file):
-                Path(version_file).touch()
+        # Add a .version file to indicate the level of versioning
+        self.write_version_file(uri, tag.name)
 
         # For Singularity this is a path, podman is a uri. If None is returned
         # there was an error and we cleanup
@@ -410,3 +414,53 @@ def install(self, name, tag=None, **kwargs):
             name = "%s:%s" % (name, tag.name)
         logger.info("Module %s was created." % name)
         return container_path
+
+    # Module software can choose how to handle each of these cases
+    def _no_default_version(self, version_file, tag):
+        return
+
+    def _module_sys_default_version(self, version_file, tag):
+        return
+
+    def _set_default_version(self, version_file, tag):
+        """
+        Set the default version to the given tag
+        """
+        template = self._load_template("default_version")
+        utils.write_file(version_file, template.render(version=tag))
+
+    def write_version_file(self, uri, latest_tag_installed=None):
+        """
+        Create the .version file, if there is a template for it.
+        """
+        version_dir = os.path.join(self.settings.module_base, uri)
+        version_file = os.path.join(version_dir, ".version")
+
+        # No default versions
+        if self.settings.default_version in [False, None]:
+            return self._no_default_version(version_file, latest_tag_installed)
+
+        # allow the module software to control versions
+        if self.settings.default_version in [True, "module_sys"]:
+            return self._module_sys_default_version(version_file, latest_tag_installed)
+
+        # First or last installed
+        if latest_tag_installed and (self.settings.default_version == "last_installed"):
+            tag = latest_tag_installed
+        else:
+            # The versions we actually have
+            found = [x for x in os.listdir(version_dir) if x != ".version"]
+            if len(found) == 1:
+                tag = found[0]
+            else:
+                if self.settings.default_version == "first_installed":
+                    selector = min
+                else:
+                    selector = max
+                tag = selector(
+                    found,
+                    key=lambda x: utils.creation_date(os.path.join(version_dir, x)),
+                )
+
+        # Write the .version file
+        return self._set_default_version(version_file, tag)

shpc/main/modules/lmod.py

@@ -3,6 +3,7 @@
 __license__ = "MPL 2.0"
 
 from .base import ModuleBase
+import shpc.utils as utils
 
 
 class Client(ModuleBase):
@@ -12,3 +13,13 @@ def __init__(self, **kwargs):
         """
         super(Client, self).__init__(**kwargs)
         self.module_extension = "lua"
+
+    def _module_sys_default_version(self, version_file, tag=None):
+        """
+        default version (default version in module_sys or True).
+        We generate a file with a non-existent version number.
+        """
+        template = self._load_template("default_version")
+        utils.write_file(version_file, template.render())
+
+    # LMOD False or null, don't generate a .version file

shpc/main/modules/tcl.py

@@ -3,6 +3,7 @@
 __license__ = "MPL 2.0"
 
 from .base import ModuleBase
+import shpc.utils as utils
 
 
 class Client(ModuleBase):
@@ -12,3 +13,13 @@ def __init__(self, **kwargs):
         """
         super(Client, self).__init__(**kwargs)
         self.module_extension = "tcl"
+
+    def _no_default_version(self, version_file, tag=None):
+        """
+        No default version (default version in False or None).
+        We generate a file with a non-existent version number.
+        """
+        template = self._load_template("default_version")
+        utils.write_file(version_file, template.render())
+
+    # TCL module_sys or True default version, don't generate a .version file

shpc/main/modules/templates/default_version

@@ -0,0 +1,2 @@
+#%Module
+set ModulesVersion "{% if version %}{{ version }}{% else %}please_specify_a_version_number{% endif %}"

shpc/main/schemas.py

@@ -129,7 +129,15 @@
     "module_name": {"type": "string"},
     "config_editor": {"type": "string"},
     "environment_file": {"type": "string"},
-    "default_version": {"type": "boolean"},
+    "default_version": {
+        "oneOf": [
+            {"type": ["null", "boolean"]},
+            {
+                "type": "string",
+                "enum": ["module_sys", "last_installed", "first_installed"],
+            },
+        ]
+    },
     "enable_tty": {"type": "boolean"},
     "wrapper_scripts": wrapper_scripts,
     "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]},

shpc/settings.yml

@@ -27,8 +27,12 @@ module_base: $root_dir/modules
 # This is where you might add a prefix to your module names, if desired.
 module_name: '{{ parsed_name.tool }}'
 
-# Create a .version file for LMOD in the module folder
-default_version: true
+# When multiple versions are available and none requested, allow module picking one itself
+# module_sys: allow the module software to decide
+# null: do nothing, versions will be specified manually
+# last_installed: use the last installed as the default
+# first_installed: use the first installed
+default_version: module_sys
 
 # store containers separately from module files
 # It's recommended to do this for faster loading

shpc/tests/test_client.py

@@ -99,9 +99,111 @@ def test_features(tmp_path, module_sys, module_file):
     assert "--nv" in content
 
 
+@pytest.mark.parametrize(
+    "default_version",
+    [True, False, "module_sys", None, "first_installed", "last_installed"],
+)
+def test_tcl_default_version(tmp_path, default_version):
+    """
+    Test tcl default versions.
+
+    True or module_sys: no .version file
+    False or None: .version file with faux number
+    first_installed: we maintain first installed version number
+    last_installed: version is updated to last installed
+    """
+    client = init_client(str(tmp_path), "tcl", "singularity")
+
+    # Customize config settings
+    client.settings.set("default_version", default_version)
+
+    # Install known tag
+    client.install("python:3.9.2-alpine")
+
+    # Get paths
+    module_dir = os.path.join(client.settings.module_base, "python")
+    version_file = os.path.join(module_dir, ".version")
+
+    if default_version in ["module_sys", True]:
+        assert not os.path.exists(version_file)
+
+    elif default_version in [False, None]:
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "please_specify_a_version_number" in content
+
+    elif default_version == "first_installed":
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+        client.install("python:3.9.5-alpine")
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+
+    elif default_version == "last_installed":
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+        client.install("python:3.9.5-alpine")
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.5-alpine" in content
+
+
+@pytest.mark.parametrize(
+    "default_version",
+    [True, False, "module_sys", None, "first_installed", "last_installed"],
+)
+def test_lmod_default_version(tmp_path, default_version):
+    """
+    Test lmod (lua) default versions.
+
+    True or module_sys: file with non-existent version number
+    False or None: no .version file
+    first_installed: we maintain first installed version number
+    last_installed: version is updated to last installed
+    """
+    client = init_client(str(tmp_path), "lmod", "singularity")
+
+    # Customize config settings
+    client.settings.set("default_version", default_version)
+
+    # Install known tag
+    client.install("python:3.9.2-alpine")
+
+    # Get paths
+    module_dir = os.path.join(client.settings.module_base, "python")
+    version_file = os.path.join(module_dir, ".version")
+
+    if default_version in ["module_sys", True]:
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "please_specify_a_version_number" in content
+
+    elif default_version in [False, None]:
+        assert not os.path.exists(version_file)
+
+    elif default_version == "first_installed":
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+        client.install("python:3.9.5-alpine")
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+
+    elif default_version == "last_installed":
+        assert os.path.exists(version_file)
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.2-alpine" in content
+        client.install("python:3.9.5-alpine")
+        content = shpc.utils.read_file(version_file)
+        assert "3.9.5-alpine" in content
+
+
 @pytest.mark.parametrize("module_sys", [("lmod"), ("tcl")])
 def test_docgen(tmp_path, module_sys):
-    """Test docgen"""
+    """
+    Test docgen
+    """
     client = init_client(str(tmp_path), module_sys, "singularity")
     client.install("python:3.9.2-slim")
     out = io.StringIO()
@@ -133,7 +235,9 @@ def test_inspect(tmp_path, module_sys, container_tech):
 
 @pytest.mark.parametrize("module_sys", [("lmod"), ("tcl")])
 def test_namespace_and_show(tmp_path, module_sys):
-    """Test namespace and show"""
+    """
+    Test namespace and show
+    """
     client = init_client(str(tmp_path), module_sys, "singularity")
     client.show("vanessa/salad:latest")
 
@@ -154,7 +258,9 @@ def test_namespace_and_show(tmp_path, module_sys):
     ],
 )
 def test_check(tmp_path, module_sys, container_tech):
-    """Test check"""
+    """
+    Test check
+    """
     client = init_client(str(tmp_path), module_sys, container_tech)
     client.install("vanessa/salad:latest")
     client.check("vanessa/salad:latest")