Merge branch 'dev' into rest-simplevm

# Conflicts:
#	bibigrid/core/rest/models.py
#	bibigrid/core/startup_rest.py

Author: XaverStiensmeier

bibigrid.yaml

@@ -1,5 +1,5 @@
-  # For an easy introduction see https://github.com/deNBI/bibigrid_clum
-  # For more detailed information see https://github.com/BiBiServ/bibigrid/blob/master/documentation/markdown/features/configuration.md
+# For an easy introduction see https://github.com/deNBI/bibigrid_clum
+# For more detailed information see https://github.com/BiBiServ/bibigrid/blob/master/documentation/markdown/features/configuration.md
 
 - # -- BEGIN: GENERAL CLUSTER INFORMATION --
   # The following options configure cluster wide keys
@@ -123,4 +123,4 @@
   #   terminate: True # whether the volume is terminated on server termination
   #   size: 50
 
-  #- [next configurations]
+  # - [next configurations]

bibigrid/core/actions/create.py

@@ -7,7 +7,6 @@
 import subprocess
 import threading
 import traceback
-from functools import partial
 
 import paramiko
 import sympy
@@ -19,46 +18,14 @@
 from bibigrid.core.utility import image_selection
 from bibigrid.core.utility.handler import ssh_handler
 from bibigrid.core.utility.paths import ansible_resources_path as a_rp
-from bibigrid.core.utility.paths import bin_path
+from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, KEY_FOLDER, CLUSTER_MEMORY_PATH
+from bibigrid.core.utility.statics.create_statics import AC_NAME, KEY_NAME, DEFAULT_SECURITY_GROUP_NAME, \
+    WIREGUARD_SECURITY_GROUP_NAME, MASTER_IDENTIFIER, WORKER_IDENTIFIER, \
+    VPNGTW_IDENTIFIER, UPLOAD_FILEPATHS
 from bibigrid.models import exceptions
 from bibigrid.models import return_threading
 from bibigrid.models.exceptions import ExecutionException, ConfigurationException
 
-PREFIX = "bibigrid"
-SEPARATOR = "-"
-PREFIX_WITH_SEP = PREFIX + SEPARATOR
-FILEPATHS = [(a_rp.PLAYBOOK_PATH, a_rp.PLAYBOOK_PATH_REMOTE), (bin_path.BIN_PATH, bin_path.BIN_PATH_REMOTE)]
-
-
-def get_identifier(identifier, cluster_id, additional=""):
-    """
-    This method does more advanced string formatting to generate master, vpngtw and worker names
-    @param identifier: master|vpngtw|worker
-    @param cluster_id: id of cluster
-    @param additional: an additional string to be added at the end
-    @return: the generated string
-    """
-    general = PREFIX_WITH_SEP + identifier + SEPARATOR + cluster_id
-    if additional or additional == 0:
-        return general + SEPARATOR + str(additional)
-    return general
-
-
-MASTER_IDENTIFIER = partial(get_identifier, identifier="master", additional="")
-WORKER_IDENTIFIER = partial(get_identifier, identifier="worker")
-VPN_WORKER_IDENTIFIER = partial(get_identifier, identifier="vpngtw")
-
-KEY_PREFIX = "tempKey_bibi"
-CONFIG_FOLDER = os.path.expanduser("~/.config/bibigrid/")
-KEY_FOLDER = os.path.join(CONFIG_FOLDER, "keys/")
-AC_NAME = "ac" + SEPARATOR + "{cluster_id}"
-KEY_NAME = KEY_PREFIX + SEPARATOR + "{cluster_id}"
-CLUSTER_MEMORY_FOLDER = KEY_FOLDER
-CLUSTER_MEMORY_FILE = ".bibigrid.mem"
-CLUSTER_MEMORY_PATH = os.path.join(CONFIG_FOLDER, CLUSTER_MEMORY_FILE)
-DEFAULT_SECURITY_GROUP_NAME = "default" + SEPARATOR + "{cluster_id}"
-WIREGUARD_SECURITY_GROUP_NAME = "wireguard" + SEPARATOR + "{cluster_id}"
-
 
 class Create:  # pylint: disable=too-many-instance-attributes,too-many-arguments
     """
@@ -103,6 +70,22 @@ def __init__(self, *, providers, configurations, config_path, log, debug=False,
             "useMasterWithPublicIp", True)
         self.log.debug("Keyname: %s", self.key_name)
 
+        os.makedirs(os.path.join(CLUSTER_INFO_FOLDER), exist_ok=True)
+        self.write_cluster_state({"floating_ip": None, "state": 202,
+                                  "message": "Create process has been started."})
+
+    def write_cluster_state(self, state):
+        state = {"cluster_id": self.cluster_id, "ssh_user": self.ssh_user, **state}
+        # last cluster
+        with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
+            yaml.safe_dump(data=state, stream=cluster_memory_file)
+        # all clusters
+        cluster_info_path = os.path.normpath(os.path.join(CLUSTER_INFO_FOLDER, f"{self.cluster_id}.yaml"))
+        if not cluster_info_path.startswith(os.path.normpath(CLUSTER_INFO_FOLDER)):
+            raise ValueError("Invalid cluster_id resulting in path traversal")
+        with open(cluster_info_path, mode="w+", encoding="UTF-8") as cluster_info_file:
+            yaml.safe_dump(data=state, stream=cluster_info_file)
+
     def create_defaults(self):
         self.log.debug("Creating default files")
         if not self.configurations[0].get("customAnsibleCfg", False) or not os.path.isfile(a_rp.ANSIBLE_CFG_PATH):
@@ -138,10 +121,6 @@ def generate_keypair(self):
         for provider in self.providers:
             provider.create_keypair(name=self.key_name, public_key=public_key)
 
-        # write cluster_id to automatically read it on following calls if no cid is given
-        with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
-            yaml.safe_dump(data={"cluster_id": self.cluster_id, "ssh_user": self.ssh_user}, stream=cluster_memory_file)
-
     def delete_old_vars(self):
         """
         Deletes host_vars and group_vars
@@ -238,14 +217,14 @@ def start_vpn_or_master(self, configuration, provider):  # pylint: disable=too-m
             raise ConfigurationException(f"MAC address for ip {configuration['private_v4']} not found.")
 
         # pylint: disable=comparison-with-callable
-        if identifier == VPN_WORKER_IDENTIFIER or (identifier == MASTER_IDENTIFIER and self.use_master_with_public_ip):
+        if identifier == VPNGTW_IDENTIFIER or (identifier == MASTER_IDENTIFIER and self.use_master_with_public_ip):
             configuration["floating_ip"] = \
                 provider.attach_available_floating_ip(network=external_network, server=server)["floating_ip_address"]
             if identifier == MASTER_IDENTIFIER:
-                with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
-                    yaml.safe_dump(
-                        data={"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"]},
-                        stream=cluster_memory_file)
+                self.write_cluster_state({"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"],
+                                          "state": 202,
+                                          "message": "Create process has been started. Master has been created."
+                                          })
             self.log.debug(f"Added floating ip {configuration['floating_ip']} to {name}.")
         elif identifier == MASTER_IDENTIFIER:
             configuration["floating_ip"] = server["private_v4"]  # pylint: enable=comparison-with-callable
@@ -303,12 +282,13 @@ def create_server_volumes(self, provider, instance, name):
         @param name: sever name
         @return:
         """
-        self.log.info("Creating volumes ...")
+        self.log.info(f"Creating volumes for {name}...")
         return_volumes = []
-
         group_instance = {"volumes": []}
         instance["group_instances"] = {name: group_instance}
+
         for i, volume in enumerate(instance.get("volumes", [])):
+            self.log.debug(f"Volume {i}: {volume}")
             if not volume.get("exists"):
                 if volume.get("permanent"):
                     infix = "perm"
@@ -332,10 +312,10 @@ def create_server_volumes(self, provider, instance, name):
                     if not return_volume:
                         raise ConfigurationException(f"Snapshot {volume['snapshot']} not found!")
                 else:
-                    self.log.debug("Creating volume...")
                     return_volume = provider.create_volume(name=volume_name, size=volume.get("size", 50),
                                                            volume_type=volume.get("type"),
                                                            description=f"Created for {name}")
+                    self.log.info(f"Volumes {i} created for {name}...")
             return_volumes.append(return_volume)
         return return_volumes
 
@@ -382,7 +362,7 @@ def prepare_vpn_or_master_args(self, configuration):
             identifier = MASTER_IDENTIFIER
         elif configuration.get("vpnInstance"):
             instance_type = configuration["vpnInstance"]
-            identifier = VPN_WORKER_IDENTIFIER
+            identifier = VPNGTW_IDENTIFIER
         else:
             self.log.warning(
                 f"Configuration {configuration['cloud_identifier']} "
@@ -464,7 +444,8 @@ def upload_data(self, private_key, clean_playbook=False):
             ssh_handler.execute_ssh(ssh_data=ssh_data, log=self.log)
         self.log.info("Uploading Data")
         ssh_data = {"floating_ip": self.master_ip, "private_key": private_key, "username": self.ssh_user,
-                    "commands": commands, "filepaths": FILEPATHS, "gateway": self.configurations[0].get("gateway", {}),
+                    "commands": commands, "filepaths": UPLOAD_FILEPATHS,
+                    "gateway": self.configurations[0].get("gateway", {}),
                     "timeout": self.ssh_timeout}
         ssh_handler.execute_ssh(ssh_data=ssh_data, log=self.log)
 
@@ -592,6 +573,9 @@ def create(self):  # pylint: disable=too-many-branches,too-many-statements
         else:
             return 0  # will be called if no exception occurred
         terminate.terminate(cluster_id=self.cluster_id, providers=self.providers, log=self.log, debug=self.debug)
+        self.write_cluster_state({"floating_ip": self.configurations[0]["floating_ip"],
+                                  "state": 500,
+                                  "message": "Cluster creation failed. Terminated remains."})
         return 1
 
     def log_cluster_start_info(self):
@@ -616,3 +600,6 @@ def log_cluster_start_info(self):
         self.log.log(42, f"Detailed cluster info: ./bibigrid.sh -i '{self.config_path}' -l -cid {self.cluster_id}")
         if self.configurations[0].get("ide"):
             self.log.log(42, f"IDE Port Forwarding: ./bibigrid.sh -i '{self.config_path}' -ide -cid {self.cluster_id}")
+        self.write_cluster_state({"floating_ip": self.configurations[0]["floating_ip"],
+                                  "state": 201,
+                                  "message": "Cluster successfully created."})

bibigrid/core/actions/list_clusters.py

@@ -6,7 +6,7 @@
 import pprint
 import re
 
-from bibigrid.core.actions import create
+from bibigrid.core.utility.statics.create_statics import MASTER_IDENTIFIER
 
 SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpngtw)-([a-zA-Z0-9]+)-\d+)$")
 
@@ -148,7 +148,7 @@ def get_master_access_ip(cluster_id, master_provider, log):
     """
     # TODO: maybe move the method from list_clusters as it is now independent of list_clusters
     log.info("Finding master ip for cluster %s...", cluster_id)
-    master = create.MASTER_IDENTIFIER(cluster_id=cluster_id)
+    master = MASTER_IDENTIFIER(cluster_id=cluster_id)
     server = master_provider.get_server(master)
     if server:
         return server.get("public_v4") or server.get("public_v6") or server.get("private_v4")

bibigrid/core/actions/terminate.py

@@ -7,10 +7,25 @@
 import re
 import time
 
-from bibigrid.core.actions import create
+import yaml
+
+from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, CLUSTER_MEMORY_PATH, KEY_FOLDER
+from bibigrid.core.utility.statics.create_statics import DEFAULT_SECURITY_GROUP_NAME, WIREGUARD_SECURITY_GROUP_NAME, \
+    KEY_NAME, AC_NAME
 from bibigrid.models.exceptions import ConflictException
 
 
+def write_cluster_state(cluster_id, state):
+    # last cluster
+    with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
+        yaml.safe_dump(data=state, stream=cluster_memory_file)
+    # all clusters
+    cluster_info_path = os.path.normpath(os.path.join(CLUSTER_INFO_FOLDER, f"{cluster_id}.yaml"))
+    if not cluster_info_path.startswith(CLUSTER_INFO_FOLDER):
+        raise ValueError("Invalid cluster_id resulting in path traversal")
+    with open(cluster_info_path, mode="w+", encoding="UTF-8") as cluster_info_file:
+        yaml.safe_dump(data=state, stream=cluster_info_file)
+
 def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
     """
     Goes through all providers and gets info of all servers which name contains cluster ID.
@@ -26,14 +41,14 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
         if not input(f"DEBUG MODE: Any non-empty input to shutdown cluster {cluster_id}. "
                      "Empty input to exit with cluster still alive:"):
             return 0
-    security_groups = [create.DEFAULT_SECURITY_GROUP_NAME]
+    security_groups = [DEFAULT_SECURITY_GROUP_NAME]
     if len(providers) > 1:
-        security_groups.append(create.WIREGUARD_SECURITY_GROUP_NAME)
+        security_groups.append(WIREGUARD_SECURITY_GROUP_NAME)
     cluster_server_state = []
     cluster_keypair_state = []
     cluster_security_group_state = []
     cluster_volume_state = []
-    tmp_keyname = create.KEY_NAME.format(cluster_id=cluster_id)
+    tmp_keyname = KEY_NAME.format(cluster_id=cluster_id)
     local_keypairs_deleted = delete_local_keypairs(tmp_keyname, log)
     if assume_yes or local_keypairs_deleted or input(
             f"WARNING: No local temporary keyfiles found for cluster {cluster_id}. "
@@ -117,7 +132,7 @@ def delete_local_keypairs(tmp_keyname, log):
     """
     success = False
     log.info("Deleting Keypair locally...")
-    tmp_keypath = os.path.join(create.KEY_FOLDER, tmp_keyname)
+    tmp_keypath = os.path.join(KEY_FOLDER, tmp_keyname)
     pub_tmp_keypath = tmp_keypath + ".pub"
     if os.path.isfile(tmp_keypath):
         os.remove(tmp_keypath)
@@ -148,14 +163,14 @@ def delete_security_groups(provider, cluster_id, security_groups, log, timeout=5
     for security_group_format in security_groups:
         security_group_name = security_group_format.format(cluster_id=cluster_id)
         attempts = 0
-        tmp_success = False
+        tmp_success = not provider.get_security_group(security_group_name)
         while not tmp_success:
             try:
-                not_found = not provider.get_security_group(security_group_name)
                 tmp_success = provider.delete_security_group(security_group_name)
             except ConflictException:
+                log.info(f"ConflictException on deletion attempt on {provider.cloud_specification['identifier']}.")
                 tmp_success = False
-            if tmp_success or not_found:
+            if tmp_success:
                 break
             if attempts < timeout:
                 attempts += 1
@@ -166,7 +181,7 @@ def delete_security_groups(provider, cluster_id, security_groups, log, timeout=5
                 log.error(f"Attempt to delete security group {security_group_name} on "
                           f"{provider.cloud_specification['identifier']} failed.")
                 break
-        log.info(f"Delete security_group {security_group_name} -> {tmp_success or not_found} on "
+        log.info(f"Delete security_group {security_group_name} -> {tmp_success} on "
                  f"{provider.cloud_specification['identifier']}.")
         success = success and tmp_success
     return success
@@ -183,7 +198,7 @@ def delete_application_credentials(master_provider, cluster_id, log):
     # implement deletion
     auth = master_provider.cloud_specification["auth"]
     if not auth.get("application_credential_id") or not auth.get("application_credential_secret"):
-        return master_provider.delete_application_credential_by_id_or_name(create.AC_NAME.format(cluster_id=cluster_id))
+        return master_provider.delete_application_credential_by_id_or_name(AC_NAME.format(cluster_id=cluster_id))
     log.info("Because you used application credentials to authenticate, "
              "no created application credentials need deletion.")
     return True
@@ -197,7 +212,7 @@ def delete_non_permanent_volumes(provider, cluster_id, log):
     @param log:
     @return: a list of the servers' (that were to be terminated) termination states
     """
-    log.info("Deleting tmp volumes on provider %s...", provider.cloud_specification['identifier'])
+    log.info("Deleting non permanent volumes on provider %s...", provider.cloud_specification['identifier'])
     volume_list = provider.list_volumes()
     cluster_volume_state = []
     volume_regex = re.compile(
@@ -228,7 +243,9 @@ def terminate_output(*, cluster_server_state, cluster_keypair_state, cluster_sec
     cluster_server_terminated = all(cluster_server_state)
     cluster_keypair_deleted = all(cluster_keypair_state)
     cluster_security_group_deleted = all(cluster_security_group_state)
-    cluster_volume_deleted = all(cluster_volume_state)
+    cluster_volume_deleted = all(all(instance_volume_states) for instance_volume_states in cluster_volume_state)
+    message = "Cluster terminated."
+    state = 200
     if cluster_existed:
         if cluster_server_terminated:
             log.info("Terminated all servers of cluster %s.", cluster_id)
@@ -248,18 +265,27 @@ def terminate_output(*, cluster_server_state, cluster_keypair_state, cluster_sec
             log.warning("Unable to delete all volumes of cluster %s.", cluster_id)
         if (cluster_server_terminated and cluster_keypair_deleted and cluster_security_group_deleted and
                 cluster_volume_deleted):
-            log.log(42, f"Successfully terminated cluster {cluster_id}.")
+            message = f"Successfully terminated cluster {cluster_id}."
+            log.log(42, message)
         else:
-            log.warning("Unable to terminate cluster %s properly."
-                        "\nAll servers terminated: %s"
-                        "\nAll keys deleted: %s"
-                        "\nAll security groups deleted: %s"
-                        "\nAll security groups deleted: %s", cluster_id, cluster_server_terminated,
-                        cluster_keypair_deleted, cluster_security_group_deleted, cluster_volume_deleted)
+            message = (f"Unable to terminate cluster {cluster_id} properly."
+                       f"\nAll servers terminated: {cluster_server_terminated}"
+                       f"\nAll keys deleted: {cluster_keypair_deleted}"
+                       f"\nAll security groups deleted: {cluster_security_group_deleted}"
+                       f"\nAll volumes deleted: {cluster_volume_deleted}")
+            log.warning(message)
         if ac_state:
             log.info("Successfully handled application credential of cluster %s.", cluster_id)
         else:
             log.warning("Unable to delete application credential of cluster %s", cluster_id)
     else:
-        log.warning("Unable to find any servers for cluster-id %s. "
-                    "Check cluster-id and configuration.\nAll keys deleted: %s", cluster_id, cluster_keypair_deleted)
+        message = "Cluster does not exist."
+        state = 404
+        log.warning(f"Unable to find any servers for cluster-id {cluster_id}. "
+                    f"Check cluster-id and configuration.\nAll keys deleted: {cluster_keypair_deleted}")
+
+    write_cluster_state(cluster_id, {"cluster_id": cluster_id,
+                                     "floating_ip": None,
+                                     "ssh_user": None,
+                                     "state": state,
+                                     "message": message})

bibigrid/core/actions/update.py

@@ -2,7 +2,6 @@
 Module that contains methods to update the master playbook
 """
 
-from bibigrid.core.actions import create
 from bibigrid.core.actions.list_clusters import dict_clusters
 from bibigrid.core.utility.handler import cluster_ssh_handler
 
@@ -19,7 +18,7 @@ def update(creator, log):
         log.warning(f"There are still workers up! {workers}")
         return 1
     if master_ip and ssh_user and used_private_key:
-        master = create.MASTER_IDENTIFIER(cluster_id=creator.cluster_id)
+        master = creator.MASTER_IDENTIFIER(cluster_id=creator.cluster_id)
         server = creator.providers[0].get_server(master)
         creator.master_ip = master_ip
         creator.configurations[0]["private_v4"] = server["private_v4"]