From a35e2bd97e1af70664108d883f01b373303217d5 Mon Sep 17 00:00:00 2001 From: Ausbeth Chiemeka Date: Thu, 26 Sep 2024 17:36:06 +0100 Subject: [PATCH 1/4] Ausbeth upgrade (#3) * Create upgrade.py * Milestone 1.1 * Fixed a bug * fixing bug * fixing bug * another bug fix * Milestone 1.1.2 * fixing bug 1.1.2 * fixing bugs 1.1.2 * fixing bugs * Trying capture stdout * fixing bugs * fixing bugs * milestone 1.1.3 * fix * fixx * another fix * and another fix * Milestone 1.2 * Milestone 1.3 * minor changes * Ensuring proper control flow of upgrade * Made uninstall function to return boolean values inorder to fix a control flow bug in upgrade.py * Using functions for cleaner code * Using functions for cleaner code 2 * upgrade all feature * fixing bug * fixing bug * upgrade preview * enhancing upgrade all * allowing user to upgrade without uninstalling old versions * typo * changing the logic of checking for the latest version since user can now keep older versions * Adding logic to check if user has modules installed first before upgrade * Replacing most print statements with logger * Making sure user does not include a version when doing shpc upgrade * fixing bug * Handling argument combinations * user-friendly message for preview * changing a preview message * Modfied warning message in main * Renaming refactoring the source code * refactoring invalid argument combination error messages * Allowing shpc upgrade software --dry-run * making logger info message clearer * Allowing shpc upgrade --all --dry-run * removing wrong comment * making logger info clearer * fixing typo in help * fixing a comment * rename refactoring * fixing unknown recipe error * changing error message for unknown recipe * fixing error message bug and cleaning code * Making shpc upgrade software -d output message clearer * removing redudant code after changing the above output message * Removing redundant arguements, adding installation of latest version to views, fixing print messages * fixing error message * removing shpc upgrade --dry-run since its similar to shpc upgrade --all --dry-run * making help message clearer * Creating test files for upgrade * refactoring inner functions within upgrade to become outer functions. For reusability outside upgrade.py * Tests for upgrading single software * fixing bug * fixing bug * fixing bug by adding force as a parameter to upgrade function * debugging * Testing * fixing bug * fixing bug * testing * debugging * debugging * testing * testing * debug statements * testing * testing * testing * testing * testing * testing * testing * testing * testing * testing * testing * testing * tests for upgrade * fixing bug * removing temporary file * removing commented out code * Removing Upgrade's --dry-run and adding it to the shpc command loop for --dry-run * Fixing issue to allow it successfully run the test * changing dry_run variable to dryrun. To keep it consistent with the rest of the code * Making incomplete command error message consistent even when user has no software installed * removing redundant code * Implementing Matthieu's suggested changes * fixing minor bug * fixing parameter and argument name (dryrun) * removing redundant code from the test * Making function "return" changes for consistency * fixing shpc linting issue * Apply automatic formatting with black and fixing SHPC linting issue * fixing lintint issue * adding upgrade tests to main test file and removing temporary test files --- shpc/client/__init__.py | 34 ++- shpc/client/help.py | 33 +++ shpc/client/upgrade.py | 190 ++++++++++++++++ shpc/main/client.py | 10 + shpc/main/container/update/docker.py | 1 - shpc/main/modules/base.py | 6 +- shpc/tests/test_client.py | 325 +++++++++++++++++++++++++++ shpc/tests/test_client.sh | 11 + 8 files changed, 606 insertions(+), 4 deletions(-) create mode 100644 shpc/client/upgrade.py diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index bd7bd2871..27557d9ba 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -324,6 +324,35 @@ def get_parser(): action="store_true", ) + # Upgrade a software to its latest version + upgrade = subparsers.add_parser( + "upgrade", + description=help.upgrade_description, + formatter_class=argparse.RawTextHelpFormatter, + ) + upgrade.add_argument( + "upgrade_recipe", + help="software to upgrade", + nargs="?", + ) + + upgrade.add_argument( + "--all", + "-a", + help="upgrade all installed software.", + dest="upgrade_all", + action="store_true", + ) + + upgrade.add_argument( + "--force", + "-f", + dest="force", + help="force upgrade without prompting for confirmation to uninstall current version(s) or install latest version to view(s).", + default=False, + action="store_true", + ) + # Update gets latest tags from OCI registries update = subparsers.add_parser( "update", @@ -377,7 +406,7 @@ def get_parser(): action="store_true", ) - for command in update, sync: + for command in update, upgrade, sync: command.add_argument( "--dry-run", "-d", @@ -400,6 +429,7 @@ def get_parser(): shell, test, uninstall, + upgrade, view, ]: command.add_argument( @@ -547,6 +577,8 @@ def help(return_code=0): from .uninstall import main elif args.command == "update": from .update import main + elif args.command == "upgrade": + from .upgrade import main elif args.command == "sync-registry": from .sync import sync_registry as main diff --git a/shpc/client/help.py b/shpc/client/help.py index b8364e476..8801ef06c 100644 --- a/shpc/client/help.py +++ b/shpc/client/help.py @@ -242,3 +242,36 @@ # Filter all modules to those with "python" $ shpc show --filter python """ + +upgrade_description = """Upgrade software to the latest version. + + # Upgrade a software to its latest version and give option to uninstall older versions or not. + # Do not include the version in the command + $ shpc upgrade quay.io/biocontainers/samtools + + # Upgrade all software + $ shpc upgrade --all + OR + $ shpc upgrade -a + + # Valid arguement combinations: + # Perform dry-run on a software to check if the latest is installed or not without upgrading it. + $ shpc upgrade quay.io/biocontainers/samtools --dry-run + OR + $ shpc upgrade quay.io/biocontainers/samtools -d + + # Perform dry-run to show version details of all installed software, to check if the latest version is installed or not without upgrading them. + $ shpc upgrade --all --dry-run + OR + $ shpc upgrade -a -d + + # Invalid arguement combinations: + $ shpc upgrade quay.io/biocontainers/samtools --all + OR + $ shpc upgrade quay.io/biocontainers/samtools -a + + $ shpc upgrade quay.io/biocontainers/samtools --all --dry-run + OR + $ shpc upgrade quay.io/biocontainers/samtools -a -d + +""" diff --git a/shpc/client/upgrade.py b/shpc/client/upgrade.py new file mode 100644 index 000000000..5238deb04 --- /dev/null +++ b/shpc/client/upgrade.py @@ -0,0 +1,190 @@ +__author__ = "Ausbeth Aguguo" +__copyright__ = "Copyright 2021-2024, Ausbeth Aguguo" +__license__ = "MPL 2.0" + +import shpc.utils as utils +from shpc.logger import logger + + +def main(args, parser, extra, subparser): + from shpc.main import get_client + + utils.ensure_no_extra(extra) + + cli = get_client(quiet=args.quiet, settings_file=args.settings_file) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) + + # Check if user entered an incomplete command + if not args.upgrade_recipe and not args.upgrade_all: + subparser.error( + "Incomplete command. The following arguments are required: upgrade_recipe, --all, or -h for more details" + ) + + # Get the list of installed software + installed_software = cli.list(return_modules=True) + + # Ensure the user has software installed before carrying out upgrade + if not installed_software: + logger.exit( + "Cannot perform shpc upgrade because you currently do not have any software installed.", + 0, + ) + + # Upgrade a specific installed software + if args.upgrade_recipe: + # Check if the provided recipe is known in any registry + try: + cli._load_container(args.upgrade_recipe) + except SystemExit: + # Give additional messages relating to shpc upgrade, to the original exit message in _load_container function + logger.exit( + "This means it cannot be upgraded because it is not installed, and cannot be installed because it is not known in any registry.\nPlease check the name or try a different recipe." + ) + # Check if the user typed an invalid argument combination + if args.upgrade_all: + logger.exit( + "Cannot use '--all' with a specific recipe. Please choose one option." + ) + # Check if the user specified a version + if ":" in args.upgrade_recipe: + logger.exit("Please use 'shpc upgrade recipe' without including a version.") + # Check if the specific software is installed + if args.upgrade_recipe not in installed_software: + logger.exit( + f"You currently do not have {args.upgrade_recipe} installed.\nYou can install it with this command: shpc install {args.upgrade_recipe}", + 0, + ) + + # Does the user just want a dry-run of the specific software? + if args.dryrun: + version_info = upgrade( + args.upgrade_recipe, cli, args, dryrun=True + ) # This returns the latest version if its available, else returns None + if version_info: + logger.info( + f"You do not have the latest version installed.\nLatest version avaiable is {version_info}" + ) + else: + logger.info( + f"You have the latest version of {args.upgrade_recipe} installed." + ) + + # Upgade the software + else: + upgrade(args.upgrade_recipe, cli, args, dryrun=False, force=args.force) + + # Upgrade all installed software + elif args.upgrade_all: + # Store the number of all outdated software + num_outdated = 0 + + # Does the user just want a dry-run of all software? + if args.dryrun: + print("Performing a dry-run on all your software...") + for software in installed_software.keys(): + version_info = upgrade(software, cli, args, dryrun=True) + if version_info: + logger.info( + f"{software} is outdated. Latest version available is {version_info}" + ) + num_outdated += 1 + else: + logger.info(f"{software} is up to date.") + # Provide a report on the dry-run + if num_outdated == 0: + logger.info("All your software are currently up to date.") + else: + logger.info(f"You have a total of {num_outdated} outdated software.") + + # Upgrade all software + else: + print("Checking your list to upgrade outdated software...") + for software in installed_software.keys(): + # Attempt upgrade on each software + upgrade_info = upgrade( + software, cli, args, dryrun=False, force=args.force + ) + # Count actual upgrades + if upgrade_info: + num_outdated += 1 + if num_outdated == 0: + logger.info("No upgrade needed. All your software are up to date.") + else: + logger.info( + f"Updated {num_outdated} outdated software from your list\nAll your software are now up to date." + ) + + +def upgrade(name, cli, args, dryrun=False, force=False): + """ + Upgrade a software to its latest version. Or preview available upgrades from the user's software list + """ + # Add namespace + name = cli.add_namespace(name) + + # Load the container configuration for the specified recipe + config = cli._load_container(name) + + # Store the installed versions and the latest version tag + installed_versions = cli.list(pattern=name, return_modules=True) + latest_version_tag = get_latest_version(name, config) + + # Compare the latest version with the user's installed version + if any(latest_version_tag in versions for versions in installed_versions.values()): + if not dryrun: + logger.info("You have the latest version of " + name + " installed already") + return None # No upgrade necessary + + else: + if dryrun: + return ( + latest_version_tag # Return the latest version for upgrade information + ) + print( + "Upgrading " + + name + + " to its latest version. Version " + + latest_version_tag + ) + + # Get the list of views the software was in + views_with_module = set() + view_dir = cli.new_module(name).module_dir + for view_name, entry in cli.views.items(): + if entry.exists(view_dir): + views_with_module.add(view_name) + + # Ask if the user wants to unintall old versions + if not cli.uninstall(name, force=force): + logger.info("Old versions of " + name + " were preserved") + + # Install the latest version + cli.install(name) + + # Install the latest version to views where the outdated version was found + if views_with_module: + msg = f"Do you also want to install the latest version of {name} to the view(s) of the previous version(s)?" + if utils.confirm_action(msg, force=force): + for view_name in views_with_module: + cli.view_install(view_name, name) + logger.info( + f"Installed the latest version of {name} to view: {view_name}" + ) + + return latest_version_tag # Upgrade occured + + +def get_latest_version(name, config): + """ + Given an added namespace of a recipe and a loaded container configuration of that namespace, + Retrieve the latest version tag. + """ + latest_version_info = config.get("latest") + if not latest_version_info: + logger.exit(f"No latest version found for {name}") + + # Extract the latest version tag + latest_version_tag = list(latest_version_info.keys())[0] + return latest_version_tag diff --git a/shpc/main/client.py b/shpc/main/client.py index 7ef602b92..a221d93f8 100644 --- a/shpc/main/client.py +++ b/shpc/main/client.py @@ -62,6 +62,16 @@ def install(self, name, tag=None, **kwargs): """ raise NotImplementedError + def upgrade(self, name, dryrun=False, force=False): + """ + Upgrade an outdated software + """ + from shpc.client.upgrade import upgrade + + cli = self + args = {} + upgrade(name, cli, args, dryrun=dryrun, force=force) + def uninstall(self, name, tag=None): """ Uninstall must also implemented by the subclass (e.g., lmod) diff --git a/shpc/main/container/update/docker.py b/shpc/main/container/update/docker.py index 35bd98077..44d8920dc 100644 --- a/shpc/main/container/update/docker.py +++ b/shpc/main/container/update/docker.py @@ -10,7 +10,6 @@ class DockerImage: - """ A thin client for getting metadata about an image. """ diff --git a/shpc/main/modules/base.py b/shpc/main/modules/base.py index fa26ba987..28a1663d7 100644 --- a/shpc/main/modules/base.py +++ b/shpc/main/modules/base.py @@ -100,14 +100,14 @@ def uninstall(self, name, force=False): # Ask before deleting anything! if not force: - msg = name + "?" + msg = "Do you wish to uninstall " + name + "?" if views_with_module: msg += ( "\nThis will uninstall the module from views:\n %s\nAre you sure?" % "\n ".join(views_with_module) ) if not utils.confirm_action(msg, force): - return + return False # If the user does not want to uninstall # Podman needs image deletion self.container.delete(module.name) @@ -149,6 +149,8 @@ def uninstall(self, name, force=False): if os.path.exists(module_dir): self.versionfile.write(module_dir) + return True # Denoting successful uninstallation + def _uninstall(self, path, base_path, name): """ Sub function, so we can pass more than one folder from uninstall diff --git a/shpc/tests/test_client.py b/shpc/tests/test_client.py index 6cdfc85cf..c2315862c 100644 --- a/shpc/tests/test_client.py +++ b/shpc/tests/test_client.py @@ -9,11 +9,14 @@ import io import os import shutil +from unittest import mock import pytest +import shpc.main.modules.views as views import shpc.main.registry as registry import shpc.utils +from shpc.client.upgrade import get_latest_version as glv from .helpers import here, init_client @@ -410,3 +413,325 @@ def test_remove(tmp_path): # Remove the module (with force) client.remove(module, force=True) assert client.registry.exists(module) is None + + +@pytest.mark.parametrize( + "module_sys, module_file, container_tech, remote, dryrun", + [ + ("lmod", "module.lua", "singularity", False, False), + ("lmod", "module.lua", "podman", False, False), + ("tcl", "module.tcl", "singularity", False, False), + ("tcl", "module.tcl", "podman", False, False), + ("lmod", "module.lua", "singularity", True, False), + ("lmod", "module.lua", "podman", True, False), + ("tcl", "module.tcl", "singularity", True, False), + ("tcl", "module.tcl", "podman", True, False), + ("lmod", "module.lua", "singularity", False, True), + ("lmod", "module.lua", "podman", False, True), + ("tcl", "module.tcl", "singularity", False, True), + ("tcl", "module.tcl", "podman", False, True), + ("lmod", "module.lua", "singularity", True, True), + ("lmod", "module.lua", "podman", True, True), + ("tcl", "module.tcl", "singularity", True, True), + ("tcl", "module.tcl", "podman", True, True), + ], +) +def test_upgrade_software_with_force( + tmp_path, module_sys, module_file, container_tech, remote, dryrun +): + """ + Test upgrading a software where uninstalling older versions and installing latest version to view is also done. + """ + client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) + + # Install an outdated version of a software + client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + + # Create the default view if it doesn't exist + view_handler = views.ViewsHandler( + settings_file=client.settings.settings_file, module_sys=module_sys + ) + assert "mpi" not in client.views + view_handler.create("mpi") + client.detect_views() + assert "mpi" in client.views + view = client.views["mpi"] + assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( + view.path + ) + assert os.path.exists(view.config_path) + assert view._config["view"]["name"] == "mpi" + assert not view._config["view"]["modules"] + assert not view._config["view"]["system_modules"] + + # Install the software to the view + client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + + # Upgrade the software to its latest version + client.upgrade("quay.io/biocontainers/samtools", dryrun=dryrun, force=True) + + # Load the container configuration for the software and get its latest version tag + name = client.add_namespace("quay.io/biocontainers/samtools") + config = client._load_container(name) + latest_version = glv(name, config) + + # Verify if the latest version of the software was installed + if not dryrun: + # Verify the module's directory exists and module files were installed + module_dir = os.path.join( + client.settings.module_base, + "quay.io/biocontainers/samtools", + latest_version, + ) + assert os.path.exists(module_dir), "Latest version directiory should exist." + module_file_path = os.path.join(module_dir, module_file) + assert os.path.exists( + module_file_path + ), "Latest version's module files should be installed." + + # Check if the older version's module directory was removed and if its module files were uninstalled + module_dir_old = os.path.join( + client.settings.module_base, + "quay.io/biocontainers/samtools", + "1.18--h50ea8bc_1", + ) + assert not os.path.exists(module_dir_old), "Older version should be uninstalled" + module_file_path = os.path.join(module_dir_old, module_file) + assert not os.path.exists( + module_file_path + ), "Older version's module files should be uninstalled." + + # Ensure the latest version was added to the view + assert client.views["mpi"].exists( + module_dir + ), "Upgraded software should be added to the view 'mpi'" + + # Verify that the latest version of the software was not installed if dry-run is TRUE + else: + module_dir = os.path.join( + client.settings.module_base, + "quay.io/biocontainers/samtools", + latest_version, + ) + assert not os.path.exists(module_dir), "Latest version should not be installed." + module_file_path = os.path.join(module_dir, module_file) + assert not os.path.exists( + module_file_path + ), "Latest version's module files should not be installed." + + +@pytest.mark.parametrize( + "module_sys,module_file,container_tech,remote", + [ + ("lmod", "module.lua", "singularity", False), + ("lmod", "module.lua", "podman", False), + ("tcl", "module.tcl", "singularity", False), + ("tcl", "module.tcl", "podman", False), + ("lmod", "module.lua", "singularity", True), + ("lmod", "module.lua", "podman", True), + ("tcl", "module.tcl", "singularity", True), + ("tcl", "module.tcl", "podman", True), + ], +) +@mock.patch("shpc.utils.confirm_action") +def test_upgrade_software_without_force( + mock_confirm_action, tmp_path, module_sys, module_file, container_tech, remote +): + """ + Test upgrading a software where uninstalling older versions and installing latest version to view is not done. + """ + client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) + + # Install an outdated version of a software + client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + + # Create the default view if it doesn't exist + view_handler = views.ViewsHandler( + settings_file=client.settings.settings_file, module_sys=module_sys + ) + assert "mpi" not in client.views + view_handler.create("mpi") + client.detect_views() + assert "mpi" in client.views + view = client.views["mpi"] + assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( + view.path + ) + assert os.path.exists(view.config_path) + assert view._config["view"]["name"] == "mpi" + assert not view._config["view"]["modules"] + assert not view._config["view"]["system_modules"] + + # Install the software to the view + client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + + # Simulate user's choice for uninstalling older versions and installing latest version to the views of the older versions + mock_confirm_action.return_value = False + + # Upgrade the software to its latest version + client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=False) + + # Load the container configuration for the software and get its latest version tag + name = client.add_namespace("quay.io/biocontainers/samtools") + config = client._load_container(name) + latest_version = glv(name, config) + + # Verify the module's directory exists and module files were installed + module_dir = os.path.join( + client.settings.module_base, "quay.io/biocontainers/samtools", latest_version + ) + assert os.path.exists(module_dir), "Latest version directiory should exist." + module_file_path = os.path.join(module_dir, module_file) + assert os.path.exists( + module_file_path + ), "Latest version's module files should be installed." + + # Ensure the older version's module directory still exists and its module files were not uninstalled + module_dir_old = os.path.join( + client.settings.module_base, + "quay.io/biocontainers/samtools", + "1.18--h50ea8bc_1", + ) + assert os.path.exists(module_dir_old), "Old version should not be uninstalled" + module_file_path = os.path.join(module_dir_old, module_file) + assert os.path.exists( + module_file_path + ), "Older version's module files should not be uninstalled." + + # Ensure the latest version was not added to the view + assert not client.views["mpi"].exists( + module_dir + ), "Upgraded software should not added to the view 'mpi'" + + +@pytest.mark.parametrize( + "module_sys,module_file,container_tech,remote", + [ + ("lmod", "module.lua", "singularity", False), + ("lmod", "module.lua", "podman", False), + ("tcl", "module.tcl", "singularity", False), + ("tcl", "module.tcl", "podman", False), + ("lmod", "module.lua", "singularity", True), + ("lmod", "module.lua", "podman", True), + ("tcl", "module.tcl", "singularity", True), + ("tcl", "module.tcl", "podman", True), + ], +) +def test_upgrade_with_latest_already_installed( + tmp_path, module_sys, module_file, container_tech, remote +): + client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) + + # Load the container configuration for the software and get is latest version + name = client.add_namespace("quay.io/biocontainers/samtools") + config = client._load_container(name) + latest_version = glv(name, config) + + # Install an outdated and the latest version of a software + client.install(f"quay.io/biocontainers/samtools:{latest_version}") + client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + + # Verify the latest version's module directory exists and module files were installed + module_dir = os.path.join( + client.settings.module_base, "quay.io/biocontainers/samtools", latest_version + ) + assert os.path.exists(module_dir), "Latest version directiory should exist." + module_file_path = os.path.join(module_dir, module_file) + assert os.path.exists(module_file_path), "Latest version module files should exist." + + # Capture the time the directory was created + module_dir_mtime_before = os.path.getmtime(module_dir) + + # Perform upgrade + client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=True) + + # Capture the time of the directory after upgrade was done + module_dir_mtime_after = os.path.getmtime(module_dir) + + # Ensure the directory did not change, to signify upgrade was not performed when the latest was already installed + assert ( + module_dir_mtime_after == module_dir_mtime_before + ), "Upgrade should not occur if latest is installed already." + + +@pytest.mark.parametrize( + "module_sys,module_file,container_tech,remote", + [ + ("lmod", "module.lua", "singularity", False), + ("lmod", "module.lua", "podman", False), + ("tcl", "module.tcl", "singularity", False), + ("tcl", "module.tcl", "podman", False), + ("lmod", "module.lua", "singularity", True), + ("lmod", "module.lua", "podman", True), + ("tcl", "module.tcl", "singularity", True), + ("tcl", "module.tcl", "podman", True), + ], +) +def test_upgrade_all_software( + tmp_path, module_sys, module_file, container_tech, remote +): + """ + Test upgrading a software where uninstalling older versions and installing latest version to view is also done. + """ + client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) + + # Install two different outdated software + client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + client.install("quay.io/biocontainers/bwa:0.7.18--he4a0461_0") + + # Create the default view if it doesn't exist + view_handler = views.ViewsHandler( + settings_file=client.settings.settings_file, module_sys=module_sys + ) + assert "mpi" not in client.views + view_handler.create("mpi") + client.detect_views() + assert "mpi" in client.views + view = client.views["mpi"] + assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( + view.path + ) + assert os.path.exists(view.config_path) + assert view._config["view"]["name"] == "mpi" + assert not view._config["view"]["modules"] + assert not view._config["view"]["system_modules"] + + # Install the software to the view + client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") + client.view_install("mpi", "quay.io/biocontainers/bwa:0.7.18--he4a0461_0") + + # Upgrade all software to thei latest version + installed_software = client.list(return_modules=True) + for software, versions in installed_software.items(): + client.upgrade(software, dryrun=False, force=True) + + # Load the container configuration for the software and get their latest version tag + name = client.add_namespace(software) + config = client._load_container(name) + latest_version = glv(name, config) + + # Verify the module's directory exists and module files were installed + module_dir = os.path.join(client.settings.module_base, software, latest_version) + assert os.path.exists(module_dir), "Latest version directiory should exist." + module_file_path = os.path.join(module_dir, module_file) + assert os.path.exists( + module_file_path + ), "Latest version's module files should be installed." + + for older_version in versions: + # Check if the older version's module directory was removed and if its module files were uninstalled + module_dir_old = os.path.join( + client.settings.module_base, software, older_version + ) + assert not os.path.exists( + module_dir_old + ), "Older version should be uninstalled" + module_file_path = os.path.join(module_dir_old, module_file) + assert not os.path.exists( + module_file_path + ), "Older version's module files should be uninstalled." + + # Ensure the latest versions were added to the view + assert client.views["mpi"].exists( + module_dir + ), "Upgraded software should be added to the view 'mpi'" diff --git a/shpc/tests/test_client.sh b/shpc/tests/test_client.sh index c7db9ece2..761745c73 100755 --- a/shpc/tests/test_client.sh +++ b/shpc/tests/test_client.sh @@ -67,6 +67,17 @@ runTest 0 $output shpc --settings-file $settings install python:3.9.2-slim runTest 0 $output shpc --settings-file $settings install vanessa/salad:latest runTest 0 $output shpc --settings-file $settings install --container_tech podman python +echo +echo "#### Testing upgrade " +runTest 0 $output shpc --settings-file $settings upgrade --help +runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/samtools:1.20--h50ea8bc_0 +runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bioconductor-bags:2.40.0--r43ha9d7317_0 +runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bwa:0.7.18--he4a0461_1 +runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --dry-run +runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --force +runTest 0 $output shpc --settings-file $settings upgrade --all --dry-run +runTest 0 $output shpc --settings-file $settings upgrade --all --force + echo echo "#### Testing get " runTest 0 $output shpc --settings-file $settings get --help From 081dc2de017f3e48d405d6edaa40aca54c4c6613 Mon Sep 17 00:00:00 2001 From: Ausbeth Chiemeka Date: Sat, 28 Sep 2024 04:28:38 +0100 Subject: [PATCH 2/4] Update upgrade.py Changing back to the default header for new file --- shpc/client/upgrade.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shpc/client/upgrade.py b/shpc/client/upgrade.py index 5238deb04..78df6b3f1 100644 --- a/shpc/client/upgrade.py +++ b/shpc/client/upgrade.py @@ -1,5 +1,5 @@ -__author__ = "Ausbeth Aguguo" -__copyright__ = "Copyright 2021-2024, Ausbeth Aguguo" +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2021-2024, Vanessa Sochat" __license__ = "MPL 2.0" import shpc.utils as utils From eb61f9fc1b8d2f04463af7c35f139418c4b2c39a Mon Sep 17 00:00:00 2001 From: Ausbeth Date: Wed, 23 Oct 2024 12:03:05 +0100 Subject: [PATCH 3/4] implementing shpc upgrade's features to shpc install --- shpc/client/__init__.py | 10 +++- shpc/client/help.py | 13 +++- shpc/client/install.py | 130 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 141 insertions(+), 12 deletions(-) diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index 27557d9ba..890f98c02 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -126,6 +126,14 @@ def get_parser(): action="store_true", ) + install.add_argument( + "--upgrade", + "-u", + help="Check if the latest version of a software is available and install it if not installed.", + dest="upgrade", + action="store_true", + ) + # List installed modules listing = subparsers.add_parser( "list", @@ -406,7 +414,7 @@ def get_parser(): action="store_true", ) - for command in update, upgrade, sync: + for command in update, upgrade, install, sync: command.add_argument( "--dry-run", "-d", diff --git a/shpc/client/help.py b/shpc/client/help.py index 8801ef06c..ceb451a5c 100644 --- a/shpc/client/help.py +++ b/shpc/client/help.py @@ -27,7 +27,7 @@ # Remove from the list $ shpc -c rm:registry:/tmp/registry""" -install_description = """Install a registry recipe. +install_description = """Install a registry recipe or upgrade an installed software. $ Install latest version $ shpc install python @@ -37,6 +37,17 @@ # Install a specific version from that set $ shpc install python:3.9.5-alpine + + # Upgrade a software to its latest version and give option to uninstall older versions or not. + # Do not include the version in the command + $ shpc install python --upgrade + OR + $ shpc install python -u + + # Perform dry-run on a software to check if the latest is installed or not without upgrading it. + $ shpc install python --upgrade --dry-run + OR + $ shpc install python -u -d """ listing_description = """List installed modules. diff --git a/shpc/client/install.py b/shpc/client/install.py index 89299db6e..bee94a194 100644 --- a/shpc/client/install.py +++ b/shpc/client/install.py @@ -3,6 +3,7 @@ __license__ = "MPL 2.0" import shpc.utils +from shpc.logger import logger def main(args, parser, extra, subparser): @@ -23,17 +24,126 @@ def main(args, parser, extra, subparser): # Update config settings on the fly cli.settings.update_params(args.config_params) - # And do the install - cli.install( - args.install_recipe, - force=args.force, - container_image=args.container_image, - keep_path=args.keep_path, - ) - if cli.settings.default_view and not args.no_view: - cli.view_install( - cli.settings.default_view, + # Get the list of the user's installed software + installed_software = cli.list(return_modules=True) + + # Upgrade a specific installed software + if args.upgrade: + # Check if the user specified a version + if ":" in args.install_recipe: + logger.exit( + "Please do not include the software version when using --upgrade argument." + ) + # Check if the specific software is installed + if args.install_recipe not in installed_software: + logger.exit( + f"You cannot carry out an upgrade on {args.install_recipe} because you do not have it installed.\nInstall it first before attempting an upgrade.", + 0, + ) + + # Does the user just want a dry-run? + if args.dryrun: + version_info = upgrade( + args.install_recipe, cli, args, dryrun=True + ) # This returns the latest version if its available, else returns None + if version_info: + logger.info( + f"You do not have the latest version installed.\nLatest version avaiable is {version_info}" + ) + else: + logger.info( + f"You have the latest version of {args.install_recipe} installed." + ) + + # Upgade the software + else: + upgrade( + args.install_recipe, + cli, + args, + dryrun=False, + ) + + # Install a new software + else: + cli.install( args.install_recipe, force=args.force, container_image=args.container_image, + keep_path=args.keep_path, ) + if cli.settings.default_view and not args.no_view: + cli.view_install( + cli.settings.default_view, + args.install_recipe, + force=args.force, + container_image=args.container_image, + ) + + +def upgrade(name, cli, args, dryrun=False): + """ + Upgrade a software to its latest version. Or preview available upgrades from the user's software list + """ + # Add namespace + name = cli.add_namespace(name) + + # Load the container configuration for the specified recipe + config = cli._load_container(name) + + # Store the installed versions and the latest version tag + installed_versions = cli.list(pattern=name, return_modules=True) + latest_version_tag = get_latest_version(name, config) + + # Compare the latest version with the user's installed version + if any(latest_version_tag in versions for versions in installed_versions.values()): + if not dryrun: + logger.info(f"You have the latest version of {name} installed already") + return None # No upgrade necessary + + else: + if dryrun: + return ( + latest_version_tag # Return the latest version for upgrade information + ) + print(f"Upgrading {name} to its latest version. Version {latest_version_tag}") + + # Get the list of views the software was in + views_with_module = set() + view_dir = cli.new_module(name).module_dir + for view_name, entry in cli.views.items(): + if entry.exists(view_dir): + views_with_module.add(view_name) + + # Ask if the user wants to unintall old versions + if not cli.uninstall(name): + logger.info(f"Old versions of {name} were preserved") + + # Install the latest version + cli.install(name) + + # Install the latest version to views where the outdated version was found + if views_with_module: + msg = f"Do you also want to install the latest version of {name} to the view(s) of the previous version(s)?" + if shpc.utils.confirm_action(msg): + for view_name in views_with_module: + cli.view_install(view_name, name) + logger.info( + f"Installed the latest version of {name} to view: {view_name}" + ) + + return latest_version_tag # Upgrade occured + + +def get_latest_version(name, config): + """ + Given an added namespace of a recipe and a loaded container configuration of that namespace, + Retrieve the latest version tag. + """ + latest_version_info = config.get("latest") + if not latest_version_info: + logger.exit(f"No latest version found for {name}") + + # Extract the latest version tag + latest_version_tag = list(latest_version_info.keys())[0] + return latest_version_tag From be22d50b1b10be76b8edb394a35ad0b09b234020 Mon Sep 17 00:00:00 2001 From: Ausbeth Date: Wed, 23 Oct 2024 12:12:40 +0100 Subject: [PATCH 4/4] Cleaning up (removing previous code on upgrade) --- shpc/client/__init__.py | 34 +--- shpc/client/help.py | 33 ---- shpc/client/upgrade.py | 190 ---------------------- shpc/main/client.py | 10 -- shpc/tests/test_client.py | 325 -------------------------------------- shpc/tests/test_client.sh | 11 -- 6 files changed, 1 insertion(+), 602 deletions(-) delete mode 100644 shpc/client/upgrade.py diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index 890f98c02..c7186862d 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -332,35 +332,6 @@ def get_parser(): action="store_true", ) - # Upgrade a software to its latest version - upgrade = subparsers.add_parser( - "upgrade", - description=help.upgrade_description, - formatter_class=argparse.RawTextHelpFormatter, - ) - upgrade.add_argument( - "upgrade_recipe", - help="software to upgrade", - nargs="?", - ) - - upgrade.add_argument( - "--all", - "-a", - help="upgrade all installed software.", - dest="upgrade_all", - action="store_true", - ) - - upgrade.add_argument( - "--force", - "-f", - dest="force", - help="force upgrade without prompting for confirmation to uninstall current version(s) or install latest version to view(s).", - default=False, - action="store_true", - ) - # Update gets latest tags from OCI registries update = subparsers.add_parser( "update", @@ -414,7 +385,7 @@ def get_parser(): action="store_true", ) - for command in update, upgrade, install, sync: + for command in update, install, sync: command.add_argument( "--dry-run", "-d", @@ -437,7 +408,6 @@ def get_parser(): shell, test, uninstall, - upgrade, view, ]: command.add_argument( @@ -585,8 +555,6 @@ def help(return_code=0): from .uninstall import main elif args.command == "update": from .update import main - elif args.command == "upgrade": - from .upgrade import main elif args.command == "sync-registry": from .sync import sync_registry as main diff --git a/shpc/client/help.py b/shpc/client/help.py index ceb451a5c..34405d2b3 100644 --- a/shpc/client/help.py +++ b/shpc/client/help.py @@ -253,36 +253,3 @@ # Filter all modules to those with "python" $ shpc show --filter python """ - -upgrade_description = """Upgrade software to the latest version. - - # Upgrade a software to its latest version and give option to uninstall older versions or not. - # Do not include the version in the command - $ shpc upgrade quay.io/biocontainers/samtools - - # Upgrade all software - $ shpc upgrade --all - OR - $ shpc upgrade -a - - # Valid arguement combinations: - # Perform dry-run on a software to check if the latest is installed or not without upgrading it. - $ shpc upgrade quay.io/biocontainers/samtools --dry-run - OR - $ shpc upgrade quay.io/biocontainers/samtools -d - - # Perform dry-run to show version details of all installed software, to check if the latest version is installed or not without upgrading them. - $ shpc upgrade --all --dry-run - OR - $ shpc upgrade -a -d - - # Invalid arguement combinations: - $ shpc upgrade quay.io/biocontainers/samtools --all - OR - $ shpc upgrade quay.io/biocontainers/samtools -a - - $ shpc upgrade quay.io/biocontainers/samtools --all --dry-run - OR - $ shpc upgrade quay.io/biocontainers/samtools -a -d - -""" diff --git a/shpc/client/upgrade.py b/shpc/client/upgrade.py deleted file mode 100644 index 78df6b3f1..000000000 --- a/shpc/client/upgrade.py +++ /dev/null @@ -1,190 +0,0 @@ -__author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2024, Vanessa Sochat" -__license__ = "MPL 2.0" - -import shpc.utils as utils -from shpc.logger import logger - - -def main(args, parser, extra, subparser): - from shpc.main import get_client - - utils.ensure_no_extra(extra) - - cli = get_client(quiet=args.quiet, settings_file=args.settings_file) - - # Update config settings on the fly - cli.settings.update_params(args.config_params) - - # Check if user entered an incomplete command - if not args.upgrade_recipe and not args.upgrade_all: - subparser.error( - "Incomplete command. The following arguments are required: upgrade_recipe, --all, or -h for more details" - ) - - # Get the list of installed software - installed_software = cli.list(return_modules=True) - - # Ensure the user has software installed before carrying out upgrade - if not installed_software: - logger.exit( - "Cannot perform shpc upgrade because you currently do not have any software installed.", - 0, - ) - - # Upgrade a specific installed software - if args.upgrade_recipe: - # Check if the provided recipe is known in any registry - try: - cli._load_container(args.upgrade_recipe) - except SystemExit: - # Give additional messages relating to shpc upgrade, to the original exit message in _load_container function - logger.exit( - "This means it cannot be upgraded because it is not installed, and cannot be installed because it is not known in any registry.\nPlease check the name or try a different recipe." - ) - # Check if the user typed an invalid argument combination - if args.upgrade_all: - logger.exit( - "Cannot use '--all' with a specific recipe. Please choose one option." - ) - # Check if the user specified a version - if ":" in args.upgrade_recipe: - logger.exit("Please use 'shpc upgrade recipe' without including a version.") - # Check if the specific software is installed - if args.upgrade_recipe not in installed_software: - logger.exit( - f"You currently do not have {args.upgrade_recipe} installed.\nYou can install it with this command: shpc install {args.upgrade_recipe}", - 0, - ) - - # Does the user just want a dry-run of the specific software? - if args.dryrun: - version_info = upgrade( - args.upgrade_recipe, cli, args, dryrun=True - ) # This returns the latest version if its available, else returns None - if version_info: - logger.info( - f"You do not have the latest version installed.\nLatest version avaiable is {version_info}" - ) - else: - logger.info( - f"You have the latest version of {args.upgrade_recipe} installed." - ) - - # Upgade the software - else: - upgrade(args.upgrade_recipe, cli, args, dryrun=False, force=args.force) - - # Upgrade all installed software - elif args.upgrade_all: - # Store the number of all outdated software - num_outdated = 0 - - # Does the user just want a dry-run of all software? - if args.dryrun: - print("Performing a dry-run on all your software...") - for software in installed_software.keys(): - version_info = upgrade(software, cli, args, dryrun=True) - if version_info: - logger.info( - f"{software} is outdated. Latest version available is {version_info}" - ) - num_outdated += 1 - else: - logger.info(f"{software} is up to date.") - # Provide a report on the dry-run - if num_outdated == 0: - logger.info("All your software are currently up to date.") - else: - logger.info(f"You have a total of {num_outdated} outdated software.") - - # Upgrade all software - else: - print("Checking your list to upgrade outdated software...") - for software in installed_software.keys(): - # Attempt upgrade on each software - upgrade_info = upgrade( - software, cli, args, dryrun=False, force=args.force - ) - # Count actual upgrades - if upgrade_info: - num_outdated += 1 - if num_outdated == 0: - logger.info("No upgrade needed. All your software are up to date.") - else: - logger.info( - f"Updated {num_outdated} outdated software from your list\nAll your software are now up to date." - ) - - -def upgrade(name, cli, args, dryrun=False, force=False): - """ - Upgrade a software to its latest version. Or preview available upgrades from the user's software list - """ - # Add namespace - name = cli.add_namespace(name) - - # Load the container configuration for the specified recipe - config = cli._load_container(name) - - # Store the installed versions and the latest version tag - installed_versions = cli.list(pattern=name, return_modules=True) - latest_version_tag = get_latest_version(name, config) - - # Compare the latest version with the user's installed version - if any(latest_version_tag in versions for versions in installed_versions.values()): - if not dryrun: - logger.info("You have the latest version of " + name + " installed already") - return None # No upgrade necessary - - else: - if dryrun: - return ( - latest_version_tag # Return the latest version for upgrade information - ) - print( - "Upgrading " - + name - + " to its latest version. Version " - + latest_version_tag - ) - - # Get the list of views the software was in - views_with_module = set() - view_dir = cli.new_module(name).module_dir - for view_name, entry in cli.views.items(): - if entry.exists(view_dir): - views_with_module.add(view_name) - - # Ask if the user wants to unintall old versions - if not cli.uninstall(name, force=force): - logger.info("Old versions of " + name + " were preserved") - - # Install the latest version - cli.install(name) - - # Install the latest version to views where the outdated version was found - if views_with_module: - msg = f"Do you also want to install the latest version of {name} to the view(s) of the previous version(s)?" - if utils.confirm_action(msg, force=force): - for view_name in views_with_module: - cli.view_install(view_name, name) - logger.info( - f"Installed the latest version of {name} to view: {view_name}" - ) - - return latest_version_tag # Upgrade occured - - -def get_latest_version(name, config): - """ - Given an added namespace of a recipe and a loaded container configuration of that namespace, - Retrieve the latest version tag. - """ - latest_version_info = config.get("latest") - if not latest_version_info: - logger.exit(f"No latest version found for {name}") - - # Extract the latest version tag - latest_version_tag = list(latest_version_info.keys())[0] - return latest_version_tag diff --git a/shpc/main/client.py b/shpc/main/client.py index a221d93f8..7ef602b92 100644 --- a/shpc/main/client.py +++ b/shpc/main/client.py @@ -62,16 +62,6 @@ def install(self, name, tag=None, **kwargs): """ raise NotImplementedError - def upgrade(self, name, dryrun=False, force=False): - """ - Upgrade an outdated software - """ - from shpc.client.upgrade import upgrade - - cli = self - args = {} - upgrade(name, cli, args, dryrun=dryrun, force=force) - def uninstall(self, name, tag=None): """ Uninstall must also implemented by the subclass (e.g., lmod) diff --git a/shpc/tests/test_client.py b/shpc/tests/test_client.py index c2315862c..6cdfc85cf 100644 --- a/shpc/tests/test_client.py +++ b/shpc/tests/test_client.py @@ -9,14 +9,11 @@ import io import os import shutil -from unittest import mock import pytest -import shpc.main.modules.views as views import shpc.main.registry as registry import shpc.utils -from shpc.client.upgrade import get_latest_version as glv from .helpers import here, init_client @@ -413,325 +410,3 @@ def test_remove(tmp_path): # Remove the module (with force) client.remove(module, force=True) assert client.registry.exists(module) is None - - -@pytest.mark.parametrize( - "module_sys, module_file, container_tech, remote, dryrun", - [ - ("lmod", "module.lua", "singularity", False, False), - ("lmod", "module.lua", "podman", False, False), - ("tcl", "module.tcl", "singularity", False, False), - ("tcl", "module.tcl", "podman", False, False), - ("lmod", "module.lua", "singularity", True, False), - ("lmod", "module.lua", "podman", True, False), - ("tcl", "module.tcl", "singularity", True, False), - ("tcl", "module.tcl", "podman", True, False), - ("lmod", "module.lua", "singularity", False, True), - ("lmod", "module.lua", "podman", False, True), - ("tcl", "module.tcl", "singularity", False, True), - ("tcl", "module.tcl", "podman", False, True), - ("lmod", "module.lua", "singularity", True, True), - ("lmod", "module.lua", "podman", True, True), - ("tcl", "module.tcl", "singularity", True, True), - ("tcl", "module.tcl", "podman", True, True), - ], -) -def test_upgrade_software_with_force( - tmp_path, module_sys, module_file, container_tech, remote, dryrun -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is also done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install an outdated version of a software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Upgrade the software to its latest version - client.upgrade("quay.io/biocontainers/samtools", dryrun=dryrun, force=True) - - # Load the container configuration for the software and get its latest version tag - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify if the latest version of the software was installed - if not dryrun: - # Verify the module's directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - latest_version, - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - # Check if the older version's module directory was removed and if its module files were uninstalled - module_dir_old = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - "1.18--h50ea8bc_1", - ) - assert not os.path.exists(module_dir_old), "Older version should be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert not os.path.exists( - module_file_path - ), "Older version's module files should be uninstalled." - - # Ensure the latest version was added to the view - assert client.views["mpi"].exists( - module_dir - ), "Upgraded software should be added to the view 'mpi'" - - # Verify that the latest version of the software was not installed if dry-run is TRUE - else: - module_dir = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - latest_version, - ) - assert not os.path.exists(module_dir), "Latest version should not be installed." - module_file_path = os.path.join(module_dir, module_file) - assert not os.path.exists( - module_file_path - ), "Latest version's module files should not be installed." - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -@mock.patch("shpc.utils.confirm_action") -def test_upgrade_software_without_force( - mock_confirm_action, tmp_path, module_sys, module_file, container_tech, remote -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is not done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install an outdated version of a software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Simulate user's choice for uninstalling older versions and installing latest version to the views of the older versions - mock_confirm_action.return_value = False - - # Upgrade the software to its latest version - client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=False) - - # Load the container configuration for the software and get its latest version tag - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify the module's directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, "quay.io/biocontainers/samtools", latest_version - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - # Ensure the older version's module directory still exists and its module files were not uninstalled - module_dir_old = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - "1.18--h50ea8bc_1", - ) - assert os.path.exists(module_dir_old), "Old version should not be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert os.path.exists( - module_file_path - ), "Older version's module files should not be uninstalled." - - # Ensure the latest version was not added to the view - assert not client.views["mpi"].exists( - module_dir - ), "Upgraded software should not added to the view 'mpi'" - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -def test_upgrade_with_latest_already_installed( - tmp_path, module_sys, module_file, container_tech, remote -): - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Load the container configuration for the software and get is latest version - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Install an outdated and the latest version of a software - client.install(f"quay.io/biocontainers/samtools:{latest_version}") - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Verify the latest version's module directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, "quay.io/biocontainers/samtools", latest_version - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists(module_file_path), "Latest version module files should exist." - - # Capture the time the directory was created - module_dir_mtime_before = os.path.getmtime(module_dir) - - # Perform upgrade - client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=True) - - # Capture the time of the directory after upgrade was done - module_dir_mtime_after = os.path.getmtime(module_dir) - - # Ensure the directory did not change, to signify upgrade was not performed when the latest was already installed - assert ( - module_dir_mtime_after == module_dir_mtime_before - ), "Upgrade should not occur if latest is installed already." - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -def test_upgrade_all_software( - tmp_path, module_sys, module_file, container_tech, remote -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is also done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install two different outdated software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - client.install("quay.io/biocontainers/bwa:0.7.18--he4a0461_0") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - client.view_install("mpi", "quay.io/biocontainers/bwa:0.7.18--he4a0461_0") - - # Upgrade all software to thei latest version - installed_software = client.list(return_modules=True) - for software, versions in installed_software.items(): - client.upgrade(software, dryrun=False, force=True) - - # Load the container configuration for the software and get their latest version tag - name = client.add_namespace(software) - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify the module's directory exists and module files were installed - module_dir = os.path.join(client.settings.module_base, software, latest_version) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - for older_version in versions: - # Check if the older version's module directory was removed and if its module files were uninstalled - module_dir_old = os.path.join( - client.settings.module_base, software, older_version - ) - assert not os.path.exists( - module_dir_old - ), "Older version should be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert not os.path.exists( - module_file_path - ), "Older version's module files should be uninstalled." - - # Ensure the latest versions were added to the view - assert client.views["mpi"].exists( - module_dir - ), "Upgraded software should be added to the view 'mpi'" diff --git a/shpc/tests/test_client.sh b/shpc/tests/test_client.sh index 761745c73..c7db9ece2 100755 --- a/shpc/tests/test_client.sh +++ b/shpc/tests/test_client.sh @@ -67,17 +67,6 @@ runTest 0 $output shpc --settings-file $settings install python:3.9.2-slim runTest 0 $output shpc --settings-file $settings install vanessa/salad:latest runTest 0 $output shpc --settings-file $settings install --container_tech podman python -echo -echo "#### Testing upgrade " -runTest 0 $output shpc --settings-file $settings upgrade --help -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/samtools:1.20--h50ea8bc_0 -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bioconductor-bags:2.40.0--r43ha9d7317_0 -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bwa:0.7.18--he4a0461_1 -runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --dry-run -runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --force -runTest 0 $output shpc --settings-file $settings upgrade --all --dry-run -runTest 0 $output shpc --settings-file $settings upgrade --all --force - echo echo "#### Testing get " runTest 0 $output shpc --settings-file $settings get --help