Merge branch 'dev' into 595/Update_Zabbix

Author: jkrue

bibigrid.yaml

@@ -1,5 +1,5 @@
-  # For an easy introduction see https://github.com/deNBI/bibigrid_clum
-  # For more detailed information see https://github.com/BiBiServ/bibigrid/blob/master/documentation/markdown/features/configuration.md
+# For an easy introduction see https://github.com/deNBI/bibigrid_clum
+# For more detailed information see https://github.com/BiBiServ/bibigrid/blob/master/documentation/markdown/features/configuration.md
 
 - # -- BEGIN: GENERAL CLUSTER INFORMATION --
   # The following options configure cluster wide keys
@@ -123,4 +123,4 @@
   #   terminate: True # whether the volume is terminated on server termination
   #   size: 50
 
-  #- [next configurations]
+  # - [next configurations]

bibigrid/core/actions/create.py

@@ -2,11 +2,11 @@
 The cluster creation (master's creation, key creation, ansible setup and execution, ...) is done here
 """
 
+import os
 import shutil
 import subprocess
 import threading
 import traceback
-import os
 
 import paramiko
 import sympy
@@ -17,13 +17,14 @@
 from bibigrid.core.utility import id_generation
 from bibigrid.core.utility import image_selection
 from bibigrid.core.utility.handler import ssh_handler
-from bibigrid.models import exceptions
-from bibigrid.models import return_threading
-from bibigrid.models.exceptions import ExecutionException, ConfigurationException
 from bibigrid.core.utility.paths import ansible_resources_path as a_rp
+from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, KEY_FOLDER, CLUSTER_MEMORY_PATH
 from bibigrid.core.utility.statics.create_statics import AC_NAME, KEY_NAME, DEFAULT_SECURITY_GROUP_NAME, \
-    WIREGUARD_SECURITY_GROUP_NAME, KEY_FOLDER, CLUSTER_MEMORY_PATH, MASTER_IDENTIFIER, WORKER_IDENTIFIER, \
+    WIREGUARD_SECURITY_GROUP_NAME, MASTER_IDENTIFIER, WORKER_IDENTIFIER, \
     VPNGTW_IDENTIFIER, UPLOAD_FILEPATHS
+from bibigrid.models import exceptions
+from bibigrid.models import return_threading
+from bibigrid.models.exceptions import ExecutionException, ConfigurationException
 
 
 class Create:  # pylint: disable=too-many-instance-attributes,too-many-arguments
@@ -69,6 +70,22 @@ def __init__(self, *, providers, configurations, config_path, log, debug=False,
             "useMasterWithPublicIp", True)
         self.log.debug("Keyname: %s", self.key_name)
 
+        os.makedirs(os.path.join(CLUSTER_INFO_FOLDER), exist_ok=True)
+        self.write_cluster_state({"floating_ip": None, "state": 202,
+                                  "message": "Create process has been started."})
+
+    def write_cluster_state(self, state):
+        state = {"cluster_id": self.cluster_id, "ssh_user": self.ssh_user, **state}
+        # last cluster
+        with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
+            yaml.safe_dump(data=state, stream=cluster_memory_file)
+        # all clusters
+        cluster_info_path = os.path.normpath(os.path.join(CLUSTER_INFO_FOLDER, f"{self.cluster_id}.yaml"))
+        if not cluster_info_path.startswith(os.path.normpath(CLUSTER_INFO_FOLDER)):
+            raise ValueError("Invalid cluster_id resulting in path traversal")
+        with open(cluster_info_path, mode="w+", encoding="UTF-8") as cluster_info_file:
+            yaml.safe_dump(data=state, stream=cluster_info_file)
+
     def create_defaults(self):
         self.log.debug("Creating default files")
         if not self.configurations[0].get("customAnsibleCfg", False) or not os.path.isfile(a_rp.ANSIBLE_CFG_PATH):
@@ -104,10 +121,6 @@ def generate_keypair(self):
         for provider in self.providers:
             provider.create_keypair(name=self.key_name, public_key=public_key)
 
-        # write cluster_id to automatically read it on following calls if no cid is given
-        with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
-            yaml.safe_dump(data={"cluster_id": self.cluster_id, "ssh_user": self.ssh_user}, stream=cluster_memory_file)
-
     def delete_old_vars(self):
         """
         Deletes host_vars and group_vars
@@ -208,10 +221,10 @@ def start_vpn_or_master(self, configuration, provider):  # pylint: disable=too-m
             configuration["floating_ip"] = \
                 provider.attach_available_floating_ip(network=external_network, server=server)["floating_ip_address"]
             if identifier == MASTER_IDENTIFIER:
-                with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
-                    yaml.safe_dump(
-                        data={"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"]},
-                        stream=cluster_memory_file)
+                self.write_cluster_state({"cluster_id": self.cluster_id, "floating_ip": configuration["floating_ip"],
+                                          "state": 202,
+                                          "message": "Create process has been started. Master has been created."
+                                          })
             self.log.debug(f"Added floating ip {configuration['floating_ip']} to {name}.")
         elif identifier == MASTER_IDENTIFIER:
             configuration["floating_ip"] = server["private_v4"]  # pylint: enable=comparison-with-callable
@@ -560,6 +573,9 @@ def create(self):  # pylint: disable=too-many-branches,too-many-statements
         else:
             return 0  # will be called if no exception occurred
         terminate.terminate(cluster_id=self.cluster_id, providers=self.providers, log=self.log, debug=self.debug)
+        self.write_cluster_state({"floating_ip": self.configurations[0]["floating_ip"],
+                                  "state": 500,
+                                  "message": "Cluster creation failed. Terminated remains."})
         return 1
 
     def log_cluster_start_info(self):
@@ -584,3 +600,6 @@ def log_cluster_start_info(self):
         self.log.log(42, f"Detailed cluster info: ./bibigrid.sh -i '{self.config_path}' -l -cid {self.cluster_id}")
         if self.configurations[0].get("ide"):
             self.log.log(42, f"IDE Port Forwarding: ./bibigrid.sh -i '{self.config_path}' -ide -cid {self.cluster_id}")
+        self.write_cluster_state({"floating_ip": self.configurations[0]["floating_ip"],
+                                  "state": 201,
+                                  "message": "Cluster successfully created."})

bibigrid/core/actions/terminate.py

@@ -7,11 +7,25 @@
 import re
 import time
 
+import yaml
+
+from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, CLUSTER_MEMORY_PATH, KEY_FOLDER
 from bibigrid.core.utility.statics.create_statics import DEFAULT_SECURITY_GROUP_NAME, WIREGUARD_SECURITY_GROUP_NAME, \
-    KEY_NAME, KEY_FOLDER, AC_NAME
+    KEY_NAME, AC_NAME
 from bibigrid.models.exceptions import ConflictException
 
 
+def write_cluster_state(cluster_id, state):
+    # last cluster
+    with open(CLUSTER_MEMORY_PATH, mode="w+", encoding="UTF-8") as cluster_memory_file:
+        yaml.safe_dump(data=state, stream=cluster_memory_file)
+    # all clusters
+    cluster_info_path = os.path.normpath(os.path.join(CLUSTER_INFO_FOLDER, f"{cluster_id}.yaml"))
+    if not cluster_info_path.startswith(CLUSTER_INFO_FOLDER):
+        raise ValueError("Invalid cluster_id resulting in path traversal")
+    with open(cluster_info_path, mode="w+", encoding="UTF-8") as cluster_info_file:
+        yaml.safe_dump(data=state, stream=cluster_info_file)
+
 def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
     """
     Goes through all providers and gets info of all servers which name contains cluster ID.
@@ -230,6 +244,8 @@ def terminate_output(*, cluster_server_state, cluster_keypair_state, cluster_sec
     cluster_keypair_deleted = all(cluster_keypair_state)
     cluster_security_group_deleted = all(cluster_security_group_state)
     cluster_volume_deleted = all(all(instance_volume_states) for instance_volume_states in cluster_volume_state)
+    message = "Cluster terminated."
+    state = 200
     if cluster_existed:
         if cluster_server_terminated:
             log.info("Terminated all servers of cluster %s.", cluster_id)
@@ -249,18 +265,27 @@ def terminate_output(*, cluster_server_state, cluster_keypair_state, cluster_sec
             log.warning("Unable to delete all volumes of cluster %s.", cluster_id)
         if (cluster_server_terminated and cluster_keypair_deleted and cluster_security_group_deleted and
                 cluster_volume_deleted):
-            log.log(42, f"Successfully terminated cluster {cluster_id}.")
+            message = f"Successfully terminated cluster {cluster_id}."
+            log.log(42, message)
         else:
-            log.warning("Unable to terminate cluster %s properly."
-                        "\nAll servers terminated: %s"
-                        "\nAll keys deleted: %s"
-                        "\nAll security groups deleted: %s"
-                        "\nAll volumes deleted: %s", cluster_id, cluster_server_terminated,
-                        cluster_keypair_deleted, cluster_security_group_deleted, cluster_volume_deleted)
+            message = (f"Unable to terminate cluster {cluster_id} properly."
+                       f"\nAll servers terminated: {cluster_server_terminated}"
+                       f"\nAll keys deleted: {cluster_keypair_deleted}"
+                       f"\nAll security groups deleted: {cluster_security_group_deleted}"
+                       f"\nAll volumes deleted: {cluster_volume_deleted}")
+            log.warning(message)
         if ac_state:
             log.info("Successfully handled application credential of cluster %s.", cluster_id)
         else:
             log.warning("Unable to delete application credential of cluster %s", cluster_id)
     else:
-        log.warning("Unable to find any servers for cluster-id %s. "
-                    "Check cluster-id and configuration.\nAll keys deleted: %s", cluster_id, cluster_keypair_deleted)
+        message = "Cluster does not exist."
+        state = 404
+        log.warning(f"Unable to find any servers for cluster-id {cluster_id}. "
+                    f"Check cluster-id and configuration.\nAll keys deleted: {cluster_keypair_deleted}")
+
+    write_cluster_state(cluster_id, {"cluster_id": cluster_id,
+                                     "floating_ip": None,
+                                     "ssh_user": None,
+                                     "state": state,
+                                     "message": message})

bibigrid/core/rest/models.py

@@ -1,17 +1,27 @@
-from typing import List, Optional
-from pydantic import BaseModel, Field
+"""
+This module contains models used by the REST api
+"""
+
+from typing import List, Optional, Literal
+from pydantic import BaseModel, Field, IPvAnyAddress
 
 
 # pylint: disable=too-few-public-methods
 
 
 class BootVolume(BaseModel):
+    """
+    Holds information about where the server boots from
+    """
     name: Optional[str]
     terminate: Optional[bool]
     size: Optional[int]
 
 
 class Volume(BaseModel):
+    """
+        Holds volume/attached storage information
+    """
     name: Optional[str]
     snapshot: Optional[str]
     permanent: Optional[bool]
@@ -24,6 +34,9 @@ class Volume(BaseModel):
 
 
 class Instance(BaseModel):
+    """
+        Holds instance/server information
+    """
     type: str
     image: str
     count: Optional[int]
@@ -35,19 +48,28 @@ class Instance(BaseModel):
 
 
 class UserRole(BaseModel):
+    """
+        Allows users to add custom ansible roles
+    """
     hosts: List[str]
     roles: List[dict]  # Replace 'dict' with more specific type if possible
     varsFiles: Optional[List[str]]
 
 
 class ElasticScheduling(BaseModel):
+    """
+        Holds info on Slurms scheduling
+    """
     SuspendTime: Optional[int]
     SuspendTimeout: Optional[int]
     ResumeTimeout: Optional[int]
     TreeWidth: Optional[int]
 
 
 class SlurmConf(BaseModel):
+    """
+    Holds info on basic Slurm settings
+    """
     db: Optional[str]
     db_user: Optional[str]
     db_password: Optional[str]
@@ -56,11 +78,17 @@ class SlurmConf(BaseModel):
 
 
 class Gateway(BaseModel):
+    """
+    Holds info regarding whether a gateway is used to connect to the master
+    """
     ip: str
     portFunction: str
 
 
 class ConfigModel(BaseModel):
+    """
+    Holds info regarding the configuration
+    """
     infrastructure: str
     cloud: str
     sshUser: str
@@ -95,6 +123,10 @@ class ConfigModel(BaseModel):
 
 
 class OtherConfigModel(ConfigModel):
+    """
+    Holds info about other configurations
+    TODO: Fill in the missing bits
+    """
     vpnInstance: Instance
 
 
@@ -141,9 +173,12 @@ class LogResponseModel(BaseModel):
     log: str
 
 
-class ReadyResponseModel(BaseModel):
+class ClusterStateResponseModel(BaseModel):
     """
-    ResponseModel for ready
+    Response model for state
     """
+    cluster_id: str
+    floating_ip: IPvAnyAddress
     message: str
-    ready: bool
+    ssh_user: str
+    state: Literal[200, 201, 204, 404, 500]

bibigrid/core/startup_rest.py

@@ -18,7 +18,7 @@
 from bibigrid.core.utility import id_generation
 from bibigrid.core.utility.handler import provider_handler
 from bibigrid.core.rest.models import ValidationResponseModel, CreateResponseModel, TerminateResponseModel, \
-    InfoResponseModel, LogResponseModel, ReadyResponseModel
+    InfoResponseModel, LogResponseModel, ClusterStateResponseModel
 
 VERSION = "0.0.1"
 DESCRIPTION = """
@@ -248,7 +248,7 @@ async def get_log(cluster_id: str, lines: int = None):
         return JSONResponse(content={"error": str(exc)}, status_code=400)
 
 
-@app.get("/bibigrid/ready/", response_model=ReadyResponseModel)
+@app.get("/bibigrid/ready/", response_model=ClusterStateResponseModel)
 async def ready(cluster_id: str):
     """
     Expects a cluster id.

bibigrid/core/utility/command_line_interpreter.py

@@ -6,8 +6,8 @@
 import logging
 import os
 
-INPUT_PATH = "~/.config/bibigrid"
-STANDARD_CONFIG_INPUT_PATH = os.path.expanduser(INPUT_PATH)
+from bibigrid.core.utility.paths.basic_path import CONFIG_FOLDER
+
 FOLDER_START = ("~/", "/")
 LOG = logging.getLogger("bibigrid")
 
@@ -39,9 +39,9 @@ def interpret_command_line():
                                                                    "Offers termination after successful create")
     parser.add_argument("-i", "--config_input", metavar="<path>", help="Path to YAML configurations file. "
                                                                        "Relative paths can be used and start "
-                                                                       f"at '{INPUT_PATH}'. "
+                                                                       f"at '{CONFIG_FOLDER}'. "
                                                                        "Required for all actions but '--version'",
-                        type=lambda s: s if s.startswith(FOLDER_START) else os.path.join(STANDARD_CONFIG_INPUT_PATH, s))
+                        type=lambda s: s if s.startswith(FOLDER_START) else os.path.join(CONFIG_FOLDER, s))
     parser.add_argument("-cid", "--cluster_id", metavar="<cluster-id>", type=check_cid, default="",
                         help="Cluster id is needed for '--ide', '--terminate_cluster' and '--update'. "
                              "If not set, last created cluster's id is used")

bibigrid/core/utility/handler/cluster_ssh_handler.py

@@ -5,7 +5,8 @@
 import os
 
 from bibigrid.core.actions import list_clusters
-from bibigrid.core.utility.statics.create_statics import KEY_FOLDER, KEY_NAME
+from bibigrid.core.utility.statics.create_statics import KEY_NAME
+from bibigrid.core.utility.paths.basic_path import KEY_FOLDER
 
 
 def get_ssh_connection_info(cluster_id, master_provider, master_configuration, log):

bibigrid/core/utility/paths/basic_path.py

@@ -9,3 +9,14 @@
 # if the relative path from this file to resources is altered, the next line must be adapted or files will not be found.
 ROOT_PATH = Path(__file__).absolute().parents[4]
 RESOURCES_PATH = os.path.join(ROOT_PATH, RESOURCES)
+
+STANDARD_CONFIG_PATH = "~/.config/bibigrid"
+CLUSTER_INFO = "cluster_info"
+CONFIG_FOLDER = os.path.expanduser(STANDARD_CONFIG_PATH)
+CLUSTER_INFO_FOLDER = os.path.join(CONFIG_FOLDER, CLUSTER_INFO)
+
+KEY_FOLDER = os.path.join(CONFIG_FOLDER, "keys/")
+
+CLUSTER_MEMORY_FOLDER = KEY_FOLDER
+CLUSTER_MEMORY_FILE = ".bibigrid.mem"
+CLUSTER_MEMORY_PATH = os.path.join(CONFIG_FOLDER, CLUSTER_MEMORY_FILE)

bibigrid/core/utility/statics/create_statics.py

@@ -2,7 +2,6 @@
 Containg static variables for create.py to avoid cyclic imports
 """
 
-import os
 from functools import partial
 
 from bibigrid.core.utility.paths import ansible_resources_path as a_rp
@@ -32,12 +31,7 @@ def get_identifier(identifier, cluster_id, additional=""):
 VPNGTW_IDENTIFIER = partial(get_identifier, identifier="vpngtw")
 
 KEY_PREFIX = "tempKey_bibi"
-CONFIG_FOLDER = os.path.expanduser("~/.config/bibigrid/")
-KEY_FOLDER = os.path.join(CONFIG_FOLDER, "keys/")
 AC_NAME = "ac" + SEPARATOR + "{cluster_id}"
 KEY_NAME = KEY_PREFIX + SEPARATOR + "{cluster_id}"
-CLUSTER_MEMORY_FOLDER = KEY_FOLDER
-CLUSTER_MEMORY_FILE = ".bibigrid.mem"
-CLUSTER_MEMORY_PATH = os.path.join(CONFIG_FOLDER, CLUSTER_MEMORY_FILE)
 DEFAULT_SECURITY_GROUP_NAME = "default" + SEPARATOR + "{cluster_id}"
 WIREGUARD_SECURITY_GROUP_NAME = "wireguard" + SEPARATOR + "{cluster_id}"

bibigrid/core/utility/validate_schema.py

@@ -49,7 +49,8 @@
      Optional('sshPublicKeyFiles'): [str], Optional('sshTimeout'): int,
      Optional('cloudScheduling'): {Optional('sshTimeout'): int}, Optional('autoMount'): bool,
      Optional('nfsShares'): [str],
-     Optional('userRoles'): [{'hosts': [str], 'roles': [{'name': str, Optional('tags'): [str]}], Optional('varsFiles'): [str]}],
+     Optional('userRoles'): [
+         {'hosts': [str], 'roles': [{'name': str, Optional('tags'): [str]}], Optional('varsFiles'): [str]}],
      Optional('localFS'): bool, Optional('localDNSlookup'): bool, Optional('slurm'): bool,
      Optional('slurmConf'): {Optional('db'): str, Optional('db_user'): str, Optional('db_password'): str,
                              Optional('munge_key'): str, Optional('elastic_scheduling'): {Optional('SuspendTime'): int,