Updating Packages Including The Version Of The OpenStack SDK (#692)

* updated packages and fixed openstack changes

* fixed tests and updated versions correctly in all files

* beginning of rework to get_free_resources

* please linter. Improve action selection process. Refactor UPPER_CASE partials to snake_case

* moved provider to contextmanager

* fixed line too long

* moved j2 template to template folder

* fixed bcrypt 5 not being compatible with latest passlib version by setting bcrypt to 4.3

* pin passlib version, too

Author: XaverStiensmeier

.pylintrc

@@ -89,7 +89,7 @@ recursive=no
 
 # When enabled, pylint would attempt to guess common misconfiguration and emit
 # user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
+# suggestion-mode=yes # deprecated
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.

bibigrid/core/actions/create.py

@@ -21,8 +21,8 @@
 from bibigrid.core.utility.paths import ansible_resources_path as a_rp
 from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, KEY_FOLDER
 from bibigrid.core.utility.statics.create_statics import AC_NAME, KEY_NAME, DEFAULT_SECURITY_GROUP_NAME, \
-    WIREGUARD_SECURITY_GROUP_NAME, MASTER_IDENTIFIER, WORKER_IDENTIFIER, \
-    VPNGTW_IDENTIFIER, UPLOAD_FILEPATHS
+    WIREGUARD_SECURITY_GROUP_NAME, master_identifier, worker_identifier, \
+    vpngtw_identifier, UPLOAD_FILEPATHS
 from bibigrid.models import exceptions
 from bibigrid.models import return_threading
 from bibigrid.models.exceptions import ExecutionException, ConfigurationException
@@ -178,7 +178,7 @@ def start_vpn_or_master(self, configuration, provider):  # pylint: disable=too-m
         """
         identifier, instance = self.prepare_vpn_or_master_args(configuration)
         external_network = provider.get_external_network(configuration["network"])
-        if identifier == MASTER_IDENTIFIER:  # pylint: disable=comparison-with-callable
+        if identifier == master_identifier:  # pylint: disable=comparison-with-callable
             name = identifier(cluster_id=self.cluster_id)
         else:
             name = identifier(cluster_id=self.cluster_id,  # pylint: disable=redundant-keyword-arg
@@ -222,17 +222,17 @@ def start_vpn_or_master(self, configuration, provider):  # pylint: disable=too-m
             raise ConfigurationException(f"MAC address for ip {configuration['private_v4']} not found.")
 
         # pylint: disable=comparison-with-callable
-        if identifier == VPNGTW_IDENTIFIER or (identifier == MASTER_IDENTIFIER and self.use_master_with_public_ip):
+        if identifier == vpngtw_identifier or (identifier == master_identifier and self.use_master_with_public_ip):
             configuration["floating_ip"] = \
                 provider.attach_available_floating_ip(network=external_network, server=server)["floating_ip_address"]
-            if identifier == MASTER_IDENTIFIER:
+            if identifier == master_identifier:
                 write_cluster_state({"cluster_id": self.cluster_id, "ssh_user": self.ssh_user,
                                      "floating_ip": configuration["floating_ip"],
                                      "state": "starting",
                                      "message": "Create process has been started. Master has been created."
                                      })
             self.log.debug(f"Added floating ip {configuration['floating_ip']} to {name}.")
-        elif identifier == MASTER_IDENTIFIER:
+        elif identifier == master_identifier:
             configuration["floating_ip"] = server["private_v4"]  # pylint: enable=comparison-with-callable
 
     def start_worker(self, worker, worker_count, configuration, provider):  # pylint: disable=too-many-locals
@@ -245,7 +245,7 @@ def start_worker(self, worker, worker_count, configuration, provider):  # pylint
         @param provider:
         @return:
         """
-        name = WORKER_IDENTIFIER(cluster_id=self.cluster_id, additional=worker_count)
+        name = worker_identifier(cluster_id=self.cluster_id, additional=worker_count)
         self.log.info(f"Starting server {name} on {provider.cloud_specification['identifier']}.")
         flavor = worker["type"]
         network = configuration["network"]
@@ -358,10 +358,10 @@ def prepare_vpn_or_master_args(self, configuration):
         """
         if configuration.get("masterInstance"):
             instance_type = configuration["masterInstance"]
-            identifier = MASTER_IDENTIFIER
+            identifier = master_identifier
         elif configuration.get("vpnInstance"):
             instance_type = configuration["vpnInstance"]
-            identifier = VPNGTW_IDENTIFIER
+            identifier = vpngtw_identifier
         else:
             self.log.warning(
                 f"Configuration {configuration['cloud_identifier']} "

bibigrid/core/actions/list_clusters.py

@@ -6,7 +6,7 @@
 import pprint
 import re
 
-from bibigrid.core.utility.statics.create_statics import MASTER_IDENTIFIER
+from bibigrid.core.utility.statics.create_statics import master_identifier
 
 SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)-([a-zA-Z0-9]+)-\d+)$")
 
@@ -148,7 +148,7 @@ def get_master_access_ip(cluster_id, master_provider, log):
     """
     # TODO: maybe move the method from list_clusters as it is now independent of list_clusters
     log.info("Finding master ip for cluster %s...", cluster_id)
-    master = MASTER_IDENTIFIER(cluster_id=cluster_id)
+    master = master_identifier(cluster_id=cluster_id)
     server = master_provider.get_server(master)
     if server:
         return server.get("public_v4") or server.get("public_v6") or server.get("private_v4")

bibigrid/core/actions/terminate.py

@@ -222,7 +222,7 @@ def delete_non_permanent_volumes(provider, cluster_id, log):
         if volume_regex.match(volume["name"]):
             log.info("Trying to delete volume %s on cloud %s.", volume['name'], provider.cloud_specification[
                 'identifier'])
-            cluster_volume_state.append(provider.delete_volume(volume))
+            cluster_volume_state.append(provider.delete_volume(volume["id"]))
     return cluster_volume_state
 
 

bibigrid/core/actions/update.py

@@ -18,7 +18,7 @@ def update(creator, log):
         log.warning(f"There are still workers up! {workers}")
         return 1
     if master_ip and ssh_user and used_private_key:
-        master = creator.MASTER_IDENTIFIER(cluster_id=creator.cluster_id)
+        master = creator.master_identifier(cluster_id=creator.cluster_id)
         server = creator.providers[0].get_server(master)
         creator.master_ip = master_ip
         creator.configurations[0]["private_v4"] = server["private_v4"]

bibigrid/core/startup.py

@@ -7,6 +7,7 @@
 import sys
 import time
 import traceback
+from contextlib import contextmanager
 
 import click
 import yaml
@@ -27,6 +28,16 @@
 logging.addLevelName(42, "PRINT")
 
 
+@contextmanager
+def managed_providers(configurations, log):
+    providers = provider_handler.get_providers(configurations, log)
+    try:
+        yield providers
+    finally:
+        for provider in providers or []:
+            provider.close()
+
+
 def get_cluster_id_from_mem():
     """
         Reads the cluster_id of the last created cluster and returns it. Used if no cluster_id is given.
@@ -62,12 +73,12 @@ def set_logger_verbosity(verbosity):
 
 def check_cid(cluster_id):
     if "-" in cluster_id:
-        new_cid = cluster_id.split("-")[-1]
-        LOG.info("-cid %s is not a cid, but probably the entire master name. Using '%s' as "
-                 "cid instead.", cluster_id, new_cid)
-        return new_cid
+        real_cid = cluster_id.split("-")[-1]
+        LOG.warning(f"-cid {cluster_id} is not a cid, but probably the entire master name. Using '{real_cid}' as "
+                    "cid instead.")
+        return real_cid
     if "." in cluster_id:
-        LOG.info("-cid %s is not a cid, but probably the master's ip. "
+        LOG.info(f"-cid {cluster_id} is not a cid, but probably the master's ip. "
                  "Using the master ip instead of cid only works if a cluster key is in your systems default ssh key "
                  "location (~/.ssh/). Otherwise bibigrid can't identify the cluster key.")
     if len(cluster_id) != id_generation.MAX_ID_LENGTH or not set(cluster_id).issubset(
@@ -98,50 +109,52 @@ def run_action(action, configurations, config_input, cluster_id, debug):
     exit_state = 0
 
     try:
-        providers = provider_handler.get_providers(configurations, LOG)
-        if not providers:
-            exit_state = 1
-
-        if providers:
-            match action:
-                case 'list':
-                    LOG.info("Action list selected")
-                    exit_state = list_clusters.log_list(cluster_id, providers, LOG)
-                case 'check':
-                    LOG.info("Action check selected")
-                    exit_state = check.check(configurations, providers, LOG)
-                case 'create':
-                    LOG.info("Action create selected")
-                    creator = create.Create(providers=providers, configurations=configurations, log=LOG, debug=debug,
-                                            config_path=config_input, cluster_id=cluster_id)
-                    LOG.log(42,
-                            "Creating a new cluster takes about 10 or more minutes depending on your cloud "
-                            "provider and your configuration. Please be patient.")
-                    exit_state = creator.create()
-                case _:
-                    if not cluster_id:
-                        cluster_id = get_cluster_id_from_mem()
-                        LOG.info("No cid (cluster_id) specified. Defaulting to last created cluster: %s",
-                                 cluster_id or 'None found')
-                    if cluster_id:
-                        LOG.debug(f"CL Argument Cluster ID: {cluster_id}")
-                        match action:
-                            case 'terminate':
-                                LOG.info("Action terminate selected")
-                                exit_state = terminate.terminate(cluster_id=cluster_id, providers=providers, log=LOG,
-                                                                 debug=debug)
-                            case 'ide':
-                                LOG.info("Action ide selected")
-                                exit_state = ide.ide(cluster_id, providers[0], configurations[0], LOG)
-                            case 'update':
-                                LOG.info("Action update selected")
-                                creator = create.Create(providers=providers, configurations=configurations, log=LOG,
-                                                        debug=debug, config_path=config_input, cluster_id=cluster_id)
-                                exit_state = update.update(creator, LOG)
-
-            for provider in providers:
-                provider.close()
-
+        with managed_providers(configurations, LOG) as providers:
+            if not providers:
+                LOG.warning("No providers found. Aborting.")
+                exit_state = 1
+                return exit_state
+
+            if action in {'terminate', 'ide', 'update'} and not cluster_id:
+                cluster_id = get_cluster_id_from_mem()
+                LOG.info(f"No cid (cluster_id) specified. Defaulting to last created cluster: "
+                         f"{cluster_id or 'None found'}")
+                if not cluster_id:
+                    LOG.warning("No cid specified and none found in memory but required. Aborting.")
+                    exit_state = 1
+                    return exit_state
+
+            if not exit_state:
+                match action:
+                    case 'list':
+                        LOG.info("Action list selected")
+                        exit_state = list_clusters.log_list(cluster_id, providers, LOG)
+                    case 'check':
+                        LOG.info("Action check selected")
+                        exit_state = check.check(configurations, providers, LOG)
+                    case 'create':
+                        LOG.info("Action create selected")
+                        creator = create.Create(providers=providers, configurations=configurations, log=LOG,
+                                                debug=debug, config_path=config_input, cluster_id=cluster_id)
+                        LOG.log(42,
+                                "Creating a new cluster takes about 10 or more minutes depending on your cloud "
+                                "provider and your configuration. Please be patient.")
+                        exit_state = creator.create()
+                    case 'terminate':
+                        LOG.info("Action terminate selected")
+                        exit_state = terminate.terminate(cluster_id=cluster_id, providers=providers, log=LOG,
+                                                         debug=debug)
+                    case 'ide':
+                        LOG.info("Action ide selected")
+                        exit_state = ide.ide(cluster_id, providers[0], configurations[0], LOG)
+                    case 'update':
+                        LOG.info("Action update selected")
+                        creator = create.Create(providers=providers, configurations=configurations, log=LOG,
+                                                debug=debug, config_path=config_input, cluster_id=cluster_id)
+                        exit_state = update.update(creator, LOG)
+                    case _:
+                        LOG.warning("Unknown Action. Aborting.")
+                        exit_state = 1
     except Exception as _:  # pylint: disable=broad-exception-caught
         exc_type, exc_value, exc_traceback = sys.exc_info()
         LOG.error("".join(traceback.format_exception(exc_type, exc_value, exc_traceback)))

bibigrid/core/utility/ansible_commands.py

@@ -60,8 +60,11 @@
 UPDATE = ("sudo apt-get update", "Update apt repository lists.")
 PYTHON3_PIP = ("sudo apt-get install -y python3-pip python3-venv", "Install python3 pip and venv using apt.")
 VENV_SETUP = ("sudo python3 -m venv /opt/bibigrid-venv", " Create bibigrid virtual environment.")
-ANSIBLE_PASSLIB = ("sudo /opt/bibigrid-venv/bin/pip install ansible==10.7 passlib",
-                   "Install Ansible 10.7 and Passlib using pip.")
+ANSIBLE_PASSLIB = (
+    "sudo /opt/bibigrid-venv/bin/pip install "
+    "ansible==10.7 passlib==1.7.4 bcrypt==4.3",
+    "Install Ansible 10.7, Passlib, and a pinned bcrypt version."
+)
 ANSIBLE_GALAXY = ("sudo /opt/bibigrid-venv/bin/ansible-galaxy collection install " +
                   "-p /usr/share/ansible/collections community.zabbix==3.2.0",
                   "Install necessary ansible-galaxy modules.")

bibigrid/core/utility/ansible_configurator.py

@@ -14,7 +14,7 @@
 from bibigrid.core.utility.handler import configuration_handler
 from bibigrid.core.utility.paths import ansible_resources_path as aRP
 from bibigrid.core.utility.paths.basic_path import ROOT_PATH
-from bibigrid.core.utility.statics.create_statics import MASTER_IDENTIFIER, VPNGTW_IDENTIFIER, WORKER_IDENTIFIER
+from bibigrid.core.utility.statics.create_statics import master_identifier, vpngtw_identifier, worker_identifier
 from bibigrid.core.utility.wireguard import wireguard_keys
 
 PYTHON_INTERPRETER = "/usr/bin/python3"
@@ -72,7 +72,7 @@ def get_full_volume_name(volume, name, count):
 def get_worker_host_vars(*, cluster_id, worker, worker_count):
     write_host_vars_remote = []
     for worker_number in range(worker.get('count', 1)):
-        name = WORKER_IDENTIFIER(cluster_id=cluster_id, additional=worker_count + worker_number)
+        name = worker_identifier(cluster_id=cluster_id, additional=worker_count + worker_number)
         write_volumes = []
         for i, volume in enumerate(worker.get("volumes", [])):
             volume_name = get_full_volume_name(volume, name, i)
@@ -86,10 +86,10 @@ def get_worker_vars(*, provider, configuration, cluster_id, worker,
                     worker_count):  # pylint: disable-msg=too-many-locals
     write_worker_vars_remote = []
     flavor_dict = provider.create_flavor_dict(flavor=worker["type"])
-    name = WORKER_IDENTIFIER(cluster_id=cluster_id,
+    name = worker_identifier(cluster_id=cluster_id,
                              additional=f"[{worker_count}-{worker_count + worker.get('count', 1) - 1}]")
     group_name = name.replace("[", "").replace("]", "").replace(":", "_").replace("-", "_")
-    regexp = WORKER_IDENTIFIER(cluster_id=cluster_id, additional=r"\d+")
+    regexp = worker_identifier(cluster_id=cluster_id, additional=r"\d+")
     partitions = worker.get("partitions", []) + [configuration["cloud_identifier"]]
     if not configuration.get("noAllPartition"):
         partitions.append("all")
@@ -124,11 +124,11 @@ def get_worker_vars(*, provider, configuration, cluster_id, worker,
 
 
 def get_vpn_vars(*, provider, configuration, cluster_id, vpngtw, vpn_count):
-    name = VPNGTW_IDENTIFIER(cluster_id=cluster_id, additional=f"{vpn_count}")
+    name = vpngtw_identifier(cluster_id=cluster_id, additional=f"{vpn_count}")
     wireguard_ip = f"10.0.0.{vpn_count + 2}"  # skipping 0 and 1 (master)
     vpn_count += 1
     flavor_dict = provider.create_flavor_dict(flavor=vpngtw["type"])
-    regexp = WORKER_IDENTIFIER(cluster_id=cluster_id, additional=r"\d+")
+    regexp = worker_identifier(cluster_id=cluster_id, additional=r"\d+")
     vpngtw_dict = {"name": name, "regexp": regexp, "image": vpngtw["image"],
                    "network": configuration["network"], "network_cidrs": configuration["subnet_cidrs"],
                    "floating_ip": configuration["floating_ip"], "private_v4": configuration["private_v4"],
@@ -144,7 +144,7 @@ def get_vpn_vars(*, provider, configuration, cluster_id, vpngtw, vpn_count):
 
 def get_master_vars(provider, configuration, cluster_id):
     master = configuration["masterInstance"]
-    name = MASTER_IDENTIFIER(cluster_id=cluster_id)
+    name = master_identifier(cluster_id=cluster_id)
     flavor_dict = provider.create_flavor_dict(flavor=master["type"])
     partitions = master.get("partitions", []) + [configuration["cloud_identifier"]]
     if not configuration.get("noAllPartition"):
@@ -269,7 +269,7 @@ def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log):  # p
     @return: ansible_hosts yaml (dict)
     """
     log.info("Generating ansible hosts file...")
-    master_name = MASTER_IDENTIFIER(cluster_id=cluster_id)
+    master_name = master_identifier(cluster_id=cluster_id)
     ansible_hosts_yaml = {"vpn": {"hosts": {},
                                   "children": {"master": {"hosts": {master_name: to_instance_host_dict(ssh_user)}},
                                                "vpngtw": {"hosts": {}}}}, "workers": {"hosts": {}, "children": {}}}
@@ -280,7 +280,7 @@ def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log):  # p
     vpngtw_count = 0
     for configuration in configurations:
         for worker in configuration.get("workerInstances", []):
-            name = WORKER_IDENTIFIER(cluster_id=cluster_id,
+            name = worker_identifier(cluster_id=cluster_id,
                                      additional=f"[{worker_count}:{worker_count + worker.get('count', 1) - 1}]")
             worker_dict = to_instance_host_dict(ssh_user, ip="")
             group_name = name.replace("[", "").replace("]", "").replace(":", "_").replace("-", "_")
@@ -290,7 +290,7 @@ def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log):  # p
             worker_count += worker.get('count', 1)
 
         if configuration.get("vpnInstance"):
-            name = VPNGTW_IDENTIFIER(cluster_id=cluster_id, additional=vpngtw_count)
+            name = vpngtw_identifier(cluster_id=cluster_id, additional=vpngtw_count)
             vpngtw_dict = to_instance_host_dict(ssh_user, ip="")
             vpngtw_dict["ansible_host"] = configuration["floating_ip"]
             vpngtws[name] = vpngtw_dict