Merge branch 'dev' into fixing-node-memory-handling

Author: XaverStiensmeier

bibigrid/core/actions/create.py

@@ -11,7 +11,6 @@
 import mergedeep
 import paramiko
 import sympy
-import yaml
 from werkzeug.utils import secure_filename
 
 from bibigrid.core.actions.terminate import delete_keypairs, delete_local_keypairs, terminate, write_cluster_state
@@ -78,6 +77,9 @@ def __init__(self, *, providers, configurations, config_path, log, debug=False,
         self.use_master_with_public_ip = not configurations[0].get("gateway") and configurations[0].get(
             "useMasterWithPublicIp", True)
         self.log.debug("Keyname: %s", self.key_name)
+        self.host_vars = {"host_entries": {}}
+        self.host_vars_lock = threading.Lock()
+        self.write_remote = []
 
         os.makedirs(os.path.join(CLUSTER_INFO_FOLDER), exist_ok=True)
         write_cluster_state(
@@ -273,14 +275,9 @@ def start_worker(self, worker, worker_count, configuration, provider):  # pylint
         # for DNS resolution an entry in the hosts file is created
         with self.worker_thread_lock:
             self.permanents.append(name)
-            with open(a_rp.HOSTS_FILE, mode="r", encoding="UTF-8") as hosts_file:
-                hosts = yaml.safe_load(hosts_file)
-            if not hosts or "host_entries" not in hosts:
-                self.log.warning("Hosts file is broken.")
-                hosts = {"host_entries": {}}
-            hosts["host_entries"][name] = server["private_v4"]
-            ansible_configurator.write_yaml(a_rp.HOSTS_FILE, hosts, self.log)
-            self.log.debug(f"Added worker {name} to hosts file {a_rp.HOSTS_FILE}.")
+            with self.host_vars_lock:
+                self.host_vars["host_entries"][name] = server["private_v4"]
+            self.log.debug(f"Added worker {name} to host vars.")
 
     # pylint: disable=duplicate-code
     def create_server_volumes(self, provider, instance, name):
@@ -359,9 +356,9 @@ def add_volume_device_info_to_instance(self, provider, server, instance):
                                f"as device {device} that is going to be mounted to "
                                f"{volume.get('mountPoint')}")
 
-            ansible_configurator.write_yaml(os.path.join(a_rp.HOST_VARS_FOLDER, f"{server['name']}.yaml"),
-                                            {"volumes": final_volumes},
-                                            self.log)
+            self.write_remote.append(
+                ({"volumes": final_volumes}, os.path.join(a_rp.HOST_VARS_FOLDER_REMOTE, f"{server['name']}.yaml"),
+                 ))
 
     def prepare_vpn_or_master_args(self, configuration):
         """
@@ -434,20 +431,20 @@ def upload_data(self, private_key, clean_playbook=False):
         @return:
         """
         self.log.debug("Running upload_data")
-        if not os.path.isfile(a_rp.HOSTS_FILE):
-            with open(a_rp.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file:
-                hosts_file.write("# placeholder file for worker DNS entries (see 003-dns)")
-
-        ansible_configurator.configure_ansible_yaml(providers=self.providers, configurations=self.configurations,
-                                                    cluster_id=self.cluster_id, log=self.log)
-        ansible_start = ssh_handler.ANSIBLE_START
-        ansible_start[-1] = (ansible_start[-1][0].format(",".join(self.permanents)), ansible_start[-1][1])
+
+        self.write_remote = (self.write_remote +
+                             ansible_configurator.configure_ansible_yaml(providers=self.providers,
+                                                                         configurations=self.configurations,
+                                                                         cluster_id=self.cluster_id,
+                                                                         log=self.log))
+        self.write_remote.append((self.host_vars, a_rp.HOSTS_FILE_REMOTE))
+        ansible_start = ssh_handler.ansible_start(",".join(self.permanents))
         self.log.debug(f"Starting playbook with {ansible_start}.")
         if self.configurations[0].get("dontUploadCredentials"):
             commands = ansible_start
         else:
             commands = [ssh_handler.get_ac_command(self.providers, AC_NAME.format(
-                cluster_id=self.cluster_id))] + ssh_handler.ANSIBLE_START
+                cluster_id=self.cluster_id))] + ansible_start
         if clean_playbook:
             self.log.info("Cleaning Playbook")
             ssh_data = {"floating_ip": self.master_ip, "private_key": private_key, "username": self.ssh_user,
@@ -456,7 +453,7 @@ def upload_data(self, private_key, clean_playbook=False):
             ssh_handler.execute_ssh(ssh_data=ssh_data, log=self.log)
         self.log.info("Uploading Data")
         ssh_data = {"floating_ip": self.master_ip, "private_key": private_key, "username": self.ssh_user,
-                    "commands": commands, "filepaths": UPLOAD_FILEPATHS,
+                    "commands": commands, "filepaths": UPLOAD_FILEPATHS, "write_remote": self.write_remote,
                     "gateway": self.configurations[0].get("gateway", {}),
                     "timeout": self.ssh_timeout}
         ssh_handler.execute_ssh(ssh_data=ssh_data, log=self.log)
@@ -469,7 +466,6 @@ def start_start_server_threads(self):
         self.log.debug("Running start_start_server_threads")
         start_server_threads = []
         worker_count = 0
-        ansible_configurator.write_yaml(a_rp.HOSTS_FILE, {"host_entries": {}}, self.log)
         for configuration, provider in zip(self.configurations, self.providers):
             start_server_thread = return_threading.ReturnThread(target=self.start_vpn_or_master,
                                                                 args=[configuration, provider])
@@ -491,7 +487,7 @@ def start_start_server_threads(self):
         for start_server_thread in start_server_threads:
             try:
                 start_server_thread.join()
-            except Exception as e: # pylint: disable=broad-except
+            except Exception as e:  # pylint: disable=broad-except
                 self.log.warning(f"Worker thread {start_server_thread} raised exception {e}.")
                 worker_exceptions.append(e)
         if worker_exceptions:

bibigrid/core/actions/version.py

@@ -14,7 +14,7 @@
 
 LOG = logging.getLogger("bibigrid")
 
-__version__ = "0.5.0"
+__version__ = "3.0.1-hotfix"
 RELEASE_DATE = "2025"
 GIT_HUB = "https://github.com/BiBiServ/bibigrid"
 

bibigrid/core/utility/ansible_commands.py

@@ -2,9 +2,6 @@
 Module containing a bunch of useful commands to be used by sshHandler.py for cluster setup
 """
 
-import os
-import bibigrid.core.utility.paths.ansible_resources_path as a_rp
-
 # TO_LOG = "| sudo tee -a /var/log/ansible.log"
 # AIY = "apt-get -y install"
 # SAU = "sudo apt-get update"
@@ -49,19 +46,16 @@
                         "Adjust playbook home permission.")
 MV_ANSIBLE_CONFIG = (
     "sudo install -D /opt/playbook/ansible.cfg /etc/ansible/ansible.cfg", "Move ansible configuration.")
-EXECUTE = (f"/opt/bibigrid-venv/bin/ansible-playbook {os.path.join(a_rp.PLAYBOOK_PATH_REMOTE, a_rp.SITE_YAML)} -i "
-           f"{os.path.join(a_rp.PLAYBOOK_PATH_REMOTE, a_rp.ANSIBLE_HOSTS)} -l {{}}",
-           "Execute ansible playbook. Be patient.")
 
 # ansible setup
 WAIT_FOR_SERVICES = (
     "while [[ $(systemctl is-active {service}) == 'active' ]]; do echo 'Waiting for service {service}'; sleep 2; done",
     "Waiting for service {service}.")
 UPDATE = ("sudo apt-get update", "Update apt repository lists.")
 PYTHON3_PIP = ("sudo apt-get install -y python3-pip python3-venv", "Install python3 pip and venv using apt.")
-VENV_SETUP = ("sudo python3 -m venv /opt/bibigrid-venv"," Create bibigrid virtual environment.")
+VENV_SETUP = ("sudo python3 -m venv /opt/bibigrid-venv", " Create bibigrid virtual environment.")
 ANSIBLE_PASSLIB = ("sudo /opt/bibigrid-venv/bin/pip install ansible==10.7 passlib",
                    "Install Ansible 10.7 and Passlib using pip.")
-ANSIBLE_GALAXY = ("sudo /opt/bibigrid-venv/bin/ansible-galaxy collection install "+
+ANSIBLE_GALAXY = ("sudo /opt/bibigrid-venv/bin/ansible-galaxy collection install " +
                   "-p /usr/share/ansible/collections community.zabbix==3.2.0",
                   "Install necessary ansible-galaxy modules.")

bibigrid/core/utility/ansible_configurator.py

@@ -56,7 +56,8 @@ def generate_site_file_yaml(user_roles):
     return site_yaml
 
 
-def write_worker_host_vars(*, cluster_id, worker, worker_count, log):
+def get_worker_host_vars(*, cluster_id, worker, worker_count):
+    write_host_vars_remote = []
     for worker_number in range(worker.get('count', 1)):
         name = WORKER_IDENTIFIER(cluster_id=cluster_id, additional=worker_count + worker_number)
         write_volumes = []
@@ -73,12 +74,14 @@ def write_worker_host_vars(*, cluster_id, worker, worker_count, log):
             else:
                 volume_name = volume["name"]
             write_volumes.append({**volume, "name": volume_name})
-        write_yaml(os.path.join(aRP.HOST_VARS_FOLDER, f"{name}.yaml"),
-                   {"volumes": write_volumes},
-                   log)
+        write_host_vars_remote.append(
+            ({"volumes": write_volumes}, os.path.join(aRP.HOST_VARS_FOLDER_REMOTE, f"{name}.yaml")))
+    return write_host_vars_remote
 
 
-def write_worker_vars(*, provider, configuration, cluster_id, worker, worker_count, log):
+def get_worker_vars(*, provider, configuration, cluster_id, worker,
+                    worker_count):  # pylint: disable-msg=too-many-locals
+    write_worker_vars_remote = []
     flavor_dict = provider.create_flavor_dict(flavor=worker["type"])
     name = WORKER_IDENTIFIER(cluster_id=cluster_id,
                              additional=f"[{worker_count}-{worker_count + worker.get('count', 1) - 1}]")
@@ -91,7 +94,8 @@ def write_worker_vars(*, provider, configuration, cluster_id, worker, worker_cou
                    "network": configuration["network"], "flavor": flavor_dict,
                    "gateway_ip": configuration["private_v4"],
                    "cloud_identifier": configuration["cloud_identifier"],
-                   "on_demand": worker.get("onDemand", True), "state": "CLOUD",
+                   "on_demand": worker.get("onDemand", True),
+                   "state": "CLOUD",
                    "partitions": partitions,
                    "boot_volume": worker.get("bootVolume", configuration.get("bootVolume", {})),
                    "meta": mergedeep.merge({}, worker.get("meta", {}), configuration.get("meta", {})),
@@ -107,15 +111,15 @@ def write_worker_vars(*, provider, configuration, cluster_id, worker, worker_cou
         worker_dict["features"] = features
 
     pass_through(configuration, worker_dict, "waitForServices", "wait_for_services")
-    write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, f"{group_name}.yaml"), worker_dict, log)
+    write_worker_vars_remote.append((worker_dict, os.path.join(aRP.GROUP_VARS_FOLDER_REMOTE, f"{group_name}.yaml")))
     if worker_dict["on_demand"]:  # not on demand instances host_vars are created in create
-        write_worker_host_vars(cluster_id=cluster_id, worker=worker, worker_count=worker_count,
-                               log=log)
+        write_worker_vars_remote = write_worker_vars_remote + get_worker_host_vars(cluster_id=cluster_id, worker=worker,
+                                                                                   worker_count=worker_count)
     worker_count += worker.get('count', 1)
-    return worker_count
+    return worker_count, write_worker_vars_remote
 
 
-def write_vpn_var(*, provider, configuration, cluster_id, vpngtw, vpn_count, log):
+def get_vpn_var(*, provider, configuration, cluster_id, vpngtw, vpn_count):
     name = VPNGTW_IDENTIFIER(cluster_id=cluster_id, additional=f"{vpn_count}")
     wireguard_ip = f"10.0.0.{vpn_count + 2}"  # skipping 0 and 1 (master)
     vpn_count += 1
@@ -131,10 +135,10 @@ def write_vpn_var(*, provider, configuration, cluster_id, vpngtw, vpn_count, log
     if configuration.get("wireguard_peer"):
         vpngtw_dict["wireguard"] = {"ip": wireguard_ip, "peer": configuration.get("wireguard_peer")}
     pass_through(configuration, vpngtw_dict, "waitForServices", "wait_for_services")
-    write_yaml(os.path.join(aRP.HOST_VARS_FOLDER, f"{name}.yaml"), vpngtw_dict, log)
+    return vpngtw_dict, os.path.join(aRP.HOST_VARS_FOLDER_REMOTE, f"{name}.yaml")
 
 
-def write_master_var(provider, configuration, cluster_id, log):
+def get_master_var(provider, configuration, cluster_id):
     master = configuration["masterInstance"]
     name = MASTER_IDENTIFIER(cluster_id=cluster_id)
     flavor_dict = provider.create_flavor_dict(flavor=master["type"])
@@ -152,10 +156,10 @@ def write_master_var(provider, configuration, cluster_id, log):
     if configuration.get("wireguard_peer"):
         master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")}
     pass_through(configuration, master_dict, "waitForServices", "wait_for_services")
-    write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, "master.yaml"), master_dict, log)
+    return master_dict, os.path.join(aRP.GROUP_VARS_FOLDER_REMOTE, "master.yaml")
 
 
-def write_host_and_group_vars(configurations, providers, cluster_id, log):
+def get_host_and_group_vars(configurations, providers, cluster_id, log):
     """
     Filters unnecessary information
     @param log:
@@ -167,17 +171,21 @@ def write_host_and_group_vars(configurations, providers, cluster_id, log):
     log.info("Generating instances file...")
     worker_count = 0
     vpn_count = 0
+    write_remote = []
     for configuration, provider in zip(configurations, providers):  # pylint: disable=too-many-nested-blocks
         for worker in configuration.get("workerInstances", []):
-            worker_count = write_worker_vars(provider=provider, configuration=configuration, cluster_id=cluster_id,
-                                             worker=worker, worker_count=worker_count, log=log)
-
+            worker_count, write_worker_vars_remote = get_worker_vars(provider=provider, configuration=configuration,
+                                                                     cluster_id=cluster_id,
+                                                                     worker=worker, worker_count=worker_count)
+            write_remote = write_remote + write_worker_vars_remote
         vpngtw = configuration.get("vpnInstance")
         if vpngtw:
-            write_vpn_var(provider=provider, configuration=configuration, cluster_id=cluster_id, vpngtw=vpngtw,
-                          vpn_count=vpn_count, log=log)
+            write_remote.append(
+                get_vpn_var(provider=provider, configuration=configuration, cluster_id=cluster_id, vpngtw=vpngtw,
+                            vpn_count=vpn_count))
         else:
-            write_master_var(provider, configuration, cluster_id, log)
+            write_remote.append(get_master_var(provider, configuration, cluster_id))
+    return write_remote
 
 
 def pass_through(dict_from, dict_to, key_from, key_to=None):
@@ -361,6 +369,8 @@ def write_yaml(path, generated_yaml, log, alias=False):
     @param log:
     @param alias:
     @return:
+    TODO: This method should be moved to a different file as it is no longer used here but at other places
+    which are not related to ansible_configurator
     """
     log.debug("Writing yaml %s", path)
 
@@ -398,21 +408,19 @@ def configure_ansible_yaml(providers, configurations, cluster_id, log):
     @return:
     """
     log.info("Writing ansible files...")
-    alias = configurations[0].get("aliasDumper", False)
     user_roles = configurations[0].get("userRoles", [])
     default_user = providers[0].cloud_specification["auth"].get("username", configurations[0].get("sshUser", "Ubuntu"))
     add_wireguard_peers(configurations)
-    for path, generated_yaml in [
-        (aRP.WORKER_SPECIFICATION_FILE, generate_worker_specification_file_yaml(configurations, log)), (
-                aRP.COMMONS_CONFIG_FILE,
-                generate_common_configuration_yaml(cidrs=get_cidrs(configurations), configurations=configurations,
-                                                   cluster_id=cluster_id, ssh_user=configurations[0]["sshUser"],
-                                                   default_user=default_user, log=log)), (aRP.HOSTS_CONFIG_FILE,
-                                                                                          generate_ansible_hosts_yaml(
-                                                                                              configurations[0][
-                                                                                                  "sshUser"],
-                                                                                              configurations,
-                                                                                              cluster_id, log)),
-        (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(user_roles))]:
-        write_yaml(path, generated_yaml, log, alias)
-    write_host_and_group_vars(configurations, providers, cluster_id, log)  # writing included in method
+    write_remote = []
+    for write_remote_tuple in [
+        (generate_worker_specification_file_yaml(configurations, log), aRP.WORKER_SPECIFICATION_FILE_REMOTE),
+        (generate_common_configuration_yaml(cidrs=get_cidrs(configurations), configurations=configurations,
+                                            cluster_id=cluster_id, ssh_user=configurations[0]["sshUser"],
+                                            default_user=default_user, log=log), aRP.COMMONS_CONFIG_FILE_REMOTE),
+        (generate_ansible_hosts_yaml(configurations[0]["sshUser"], configurations, cluster_id, log),
+         aRP.HOSTS_CONFIG_FILE_REMOTE),
+        (generate_site_file_yaml(user_roles), aRP.SITE_CONFIG_FILE_REMOTE)]:
+        write_remote.append(write_remote_tuple)
+    write_remote = write_remote + get_host_and_group_vars(configurations, providers, cluster_id,
+                                                          log)
+    return write_remote