709 allow selection of pre existing floating ip (#711)

* fixed gres and included installation of NVIDIA drivers

* Fixed gres, zabbix repository and logging

Untested. Includes installation for nvidia

* removed pausing line

* support new rest schema

* updated gres and volume documentation

* allow selection of floating ip by id

now a specific floating ip can be assigned to master or vpn by setting the floatingIpId key in the configuration.

#709

* prevent preexisting fip deletion

now floating ips are no longer released on clouds where they are defined as pre-existing

#709

* check floating ip existence

added _check_floating_ip to verify the existence of the floating ip

#705

* update provider tests

updated provider tests to cover new api openstack api calls that had not been used in the past

#709

* fixed tests

some tests failed due to unexpected new parameters from the new floating_ip check on terminate

#709

* fix remaining floating ip

apparently when you create a floating ip attached to a server and delete that server quickly, the floating ip remains.

#709

* fix terminate for REST

REST's terminate did not pass the floating ip list. Now it is passed as well.

#709

* pleases linter

#709

* adds documentation for floatingIpId

#709

---------

Co-authored-by: Jan Krüger <jkrue@users.noreply.github.com>

Author: XaverStiensmeier

bibigrid.yaml

@@ -27,6 +27,7 @@
 
   ## If you use a gateway or start a cluster from the cloud, your master does not need a public ip.
   # useMasterWithPublicIp: False # defaults True if False no public-ip (floating-ip) will be allocated
+  # floatingIpId: # if set, uses existing floating ip which will not be deleted on termination.
   # gateway: # if you want to use a gateway for create.
   # ip: # IP of gateway to use
   # portFunction: 30000 + oct4 # variables are called: oct1.oct2.oct3.oct4

bibigrid/core/actions/create.py

@@ -17,7 +17,7 @@
 from bibigrid.core.utility import ansible_configurator
 from bibigrid.core.utility import id_generation
 from bibigrid.core.utility import image_selection
-from bibigrid.core.utility.handler import ssh_handler
+from bibigrid.core.utility.handler import ssh_handler, configuration_handler
 from bibigrid.core.utility.paths import ansible_resources_path as a_rp
 from bibigrid.core.utility.paths.basic_path import CLUSTER_INFO_FOLDER, KEY_FOLDER
 from bibigrid.core.utility.statics.create_statics import AC_NAME, KEY_NAME, DEFAULT_SECURITY_GROUP_NAME, \
@@ -223,8 +223,20 @@ def start_vpn_or_master(self, configuration, provider):  # pylint: disable=too-m
 
         # pylint: disable=comparison-with-callable
         if identifier == vpngtw_identifier or (identifier == master_identifier and self.use_master_with_public_ip):
-            configuration["floating_ip"] = \
-                provider.attach_available_floating_ip(network=external_network, server=server)["floating_ip_address"]
+            if not configuration.get("floatingIpId"):
+                configuration["floating_ip"] = provider.create_floating_ip(network=external_network, server=server)[
+                    "floating_ip_address"]
+            else:
+                floating_ip = provider.get_floating_ip(configuration["floatingIpId"])
+                if not floating_ip:
+                    raise ValueError(
+                        f"Floating ip {configuration['floatingIpId']} does not exist on "
+                        f"{provider.cloud_specification['identifier']}!")
+                configuration["floating_ip"] = floating_ip["floating_ip_address"]
+                _ = provider.add_ip_list(server=server, floating_ip=[configuration["floating_ip"]])
+            self.log.info(
+                f"Server {name} uses floating ip {configuration['floating_ip']} ({'pre-existing'
+                if configuration.get('floatingIpId') else 'created'})")
             if identifier == master_identifier:
                 write_cluster_state({"cluster_id": self.cluster_id, "ssh_user": self.ssh_user,
                                      "floating_ip": configuration["floating_ip"],
@@ -546,8 +558,9 @@ def create(self):  # pylint: disable=too-many-branches,too-many-statements
                 delete_local_keypairs(tmp_keyname=self.key_name, log=self.log)
             if self.debug:
                 self.log.info("DEBUG MODE: Entering termination...")
-                terminate(cluster_id=self.cluster_id, providers=self.providers, debug=self.debug,
-                          log=self.log)
+                terminate(cluster_id=self.cluster_id, providers=self.providers,
+                          floating_ip_ids=configuration_handler.get_list_by_key(self.configurations, "floatingIpId"),
+                          log=self.log, debug=self.debug)
         except exceptions.ConnectionException:
             self.log.error(traceback.format_exc())
             self.log.error("Connection couldn't be established. Check Provider connection.")
@@ -577,7 +590,9 @@ def create(self):  # pylint: disable=too-many-branches,too-many-statements
             self.log.error(f"Unexpected error: '{str(exc)}' ({type(exc)}) Contact a developer!)")
         else:
             return 0  # will be called if no exception occurred
-        terminate(cluster_id=self.cluster_id, providers=self.providers, log=self.log, debug=self.debug)
+        terminate(cluster_id=self.cluster_id, providers=self.providers,
+                  floating_ip_ids=configuration_handler.get_list_by_key(self.configurations, "floatingIpId"),
+                  log=self.log, debug=self.debug)
         write_cluster_state({"cluster_id": self.cluster_id, "ssh_user": self.ssh_user,
                              "floating_ip": self.configurations[0].get("floating_ip"),
                              "state": "failed",

bibigrid/core/actions/terminate.py

@@ -29,7 +29,8 @@ def write_cluster_state(state):
         yaml.safe_dump(data=state, stream=cluster_info_file)
 
 
-def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
+# pylint: disable=too-many-locals
+def terminate(*, cluster_id, providers, floating_ip_ids, log, debug=False, assume_yes=False):
     """
     Goes through all providers and gets info of all servers which name contains cluster ID.
     It then checks if any resources are reserved, but not used and frees them that were hold by the cluster.
@@ -58,9 +59,9 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
             f"This might not be your cluster. Are you sure you want to terminate it?\n"
             f"Any non-empty input to shutdown cluster {cluster_id}. "
             f"Empty input to exit with cluster still alive:"):
-        for provider in providers:
+        for provider, floating_ip_id in zip(providers, floating_ip_ids):
             log.info("Terminating cluster %s on cloud %s", cluster_id, provider.cloud_specification['identifier'])
-            cluster_server_state += terminate_servers(cluster_id, provider, log)
+            cluster_server_state += terminate_servers(cluster_id, provider, floating_ip_id, log)
             cluster_keypair_state.append(delete_keypairs(provider, tmp_keyname, log))
             cluster_security_group_state.append(delete_security_groups(provider, cluster_id, security_groups, log))
             cluster_volume_state.append(delete_non_permanent_volumes(provider, cluster_id, log))
@@ -72,11 +73,12 @@ def terminate(cluster_id, providers, log, debug=False, assume_yes=False):
     return 0
 
 
-def terminate_servers(cluster_id, provider, log):
+def terminate_servers(cluster_id, provider, floating_ip_id, log):
     """
     Terminates all servers that match the bibigrid regex.
     @param cluster_id: id of cluster to terminate
     @param provider: provider that holds all servers in server_list
+    @param floating_ip_id: floating ip id of master/vpn
     @param log:
     @return: a list of the servers' (that were to be terminated) termination states
     """
@@ -89,19 +91,20 @@ def terminate_servers(cluster_id, provider, log):
         if server_regex.match(server["name"]):
             log.info("Trying to terminate Server %s on cloud %s.", server['name'],
                      provider.cloud_specification['identifier'])
-            cluster_server_state.append(terminate_server(provider, server, log))
+            cluster_server_state.append(terminate_server(provider, server, not bool(floating_ip_id), log))
     return cluster_server_state
 
 
-def terminate_server(provider, server, log):
+def terminate_server(provider, server, delete_ips, log):
     """
     Terminates a single server and stores the termination state
     @param provider: the provider that holds the server.
     @param server: the server that is to be terminated
+    @param delete_ips: if true all floating ips of the server are released
     @param log:
     @return: true if the server has been terminated, false else
     """
-    terminated = provider.delete_server(server["id"])
+    terminated = provider.delete_server(server["id"], delete_ips=delete_ips)
     if not terminated:
         log.warning("Unable to terminate server %s on provider %s.", server['name'],
                     provider.cloud_specification['identifier'])

bibigrid/core/provider.py

@@ -203,14 +203,43 @@ def get_external_network(self, network_name_or_id):
         """
 
     @abstractmethod
-    def attach_available_floating_ip(self, network=None, server=None):
+    def add_ip_list(self, *, server, ips, wait=False, timeout=60, fixed_address=None, nat_destination=None):
         """
-        Get a floating IP from a network or a pool and attach it to the server
+        Add ip list to server.
+        :param server:
+        :param ips:
+        :param wait:
+        :param timeout:
+        :param fixed_address:
+        :param nat_destination:
+        :return:
+        """
+
+    @abstractmethod
+    def get_floating_ip(self, floating_ip_id, filters=None):
+        """
+        Get a floating IP by id.
+        @param floating_ip_id:
+        @param filters:
+        @return:
+        """
+
+    @abstractmethod
+    def create_floating_ip(self, *, network=None, server=None, fixed_address=None, nat_destination=None, port=None,
+                           wait=False, timeout=60):
+        """
+
         @param network:
         @param server:
+        @param fixed_address:
+        @param nat_destination:
+        @param port:
+        @param wait:
+        @param timeout:
         @return:
         """
 
+
     @abstractmethod
     def get_images(self):
         """

bibigrid/core/startup.py

@@ -142,8 +142,10 @@ def run_action(action, configurations, config_input, cluster_id, debug):
                         exit_state = creator.create()
                     case 'terminate':
                         LOG.info("Action terminate selected")
-                        exit_state = terminate.terminate(cluster_id=cluster_id, providers=providers, log=LOG,
-                                                         debug=debug)
+                        exit_state = terminate.terminate(cluster_id=cluster_id, providers=providers,
+                                                         floating_ip_ids=configuration_handler.get_list_by_key(
+                                                             configurations=configurations, key="floatingIpId"),
+                                                         log=LOG, debug=debug)
                     case 'ide':
                         LOG.info("Action ide selected")
                         exit_state = ide.ide(cluster_id, providers[0], configurations[0], LOG)

bibigrid/core/startup_rest.py

@@ -193,7 +193,9 @@ async def terminate_cluster(cluster_id: str,
 
     try:
         providers = provider_handler.get_providers(configurations, log)
-        thread = threading.Thread(target=terminate.terminate, args=(cluster_id, providers, log))
+        thread = threading.Thread(target=terminate.terminate,
+                                  args=(cluster_id, providers, configuration_handler.get_list_by_key(
+                                      configurations=configurations, key="floatingIpId"), log))
         thread.start()
         return JSONResponse(content={"message": "Termination successfully requested."}, status_code=202)
     except Exception as exc:  # pylint: disable=broad-except

bibigrid/core/utility/handler/configuration_handler.py

@@ -42,15 +42,15 @@ def read_configuration(log, path, configuration_list=True):
     return configuration
 
 
-def get_list_by_key(configurations, key, get_empty=True):
+def get_list_by_key(configurations, key, include_none=True):
     """
     Returns a list of objects which are value to the key.
-    @param get_empty: if true empty configurations return None
+    @param include_none: if true empty configurations return None
     @param configurations: YAML of configuration File containing the configuration-data for each provider
     @param key: Key that is looked out for
     @return: List of values of said key through all configs
     """
-    return [configuration.get(key) for configuration in configurations if configuration.get(key) or get_empty]
+    return [configuration.get(key) for configuration in configurations if configuration.get(key) or include_none]
 
 
 # def get_dict_list_by_key_list(configurations, keys, get_empty=True):

bibigrid/core/utility/validate_configuration.py

@@ -214,6 +214,7 @@ def validate(self):
         checks = [("master/vpn", self._check_master_vpn_worker), ("servergroup", self._check_server_groups),
                   ("instances", self._check_instances), ("volumes", self._check_volumes),
                   ("network", self._check_network),
+                  ("floatingIp", self._check_floating_ip),
                   ("quotas", self._check_quotas), ("sshPublicKeyFiles", self._check_ssh_public_key_files),
                   ("cloudYamls", self.check_clouds_yamls), ("nfs", self._check_nfs),
                   ("global security groups", self._check_configurations_security_groups)]
@@ -222,6 +223,19 @@ def validate(self):
                 success = evaluate(check_name, check_function(), self.log) and success
         return success
 
+    def _check_floating_ip(self):
+        success = True
+        for configuration, provider in zip(self.configurations, self.providers):
+            if configuration.get("floatingIpId"):
+                floating_ip = provider.get_floating_ip(id=configuration["floatingIpId"])
+                if not floating_ip:
+                    self.log.warning(f"Couldn't find floating ip {configuration["floatingIpId"]} on cloud "
+                                     f"{provider.cloud_specification['identifier']}.")
+                    success = False
+                else:
+                    self.log.debug(f"Found {floating_ip} on cloud {provider.cloud_specification['identifier']}.")
+        return success
+
     def _check_security_groups(self, provider, security_groups):
         success = True
         if not security_groups:

bibigrid/models/configuration.py

@@ -137,6 +137,7 @@ class BaseConfig(StrictModel):
     sshUser: str
     subnet: Optional[str] = Field(default=None)
     network: Optional[str] = Field(default=None)
+    floatingIpId: Optional[str] = Field(default=None)
     securityGroups: Optional[list[str]] = Field(default_factory=list)
     serverGroup: Optional[str] = None
     waitForServices: Optional[List[str]] = Field(default_factory=list)

bibigrid/openstack/openstack_provider.py

@@ -288,16 +288,45 @@ def get_external_network(self, network_name_or_id):
                     return router.external_gateway_info["network_id"]
         return None
 
-    def attach_available_floating_ip(self, network=None, server=None):
+    def get_floating_ip(self, floating_ip_id, filters=None):
         """
-        Get a floating IP from a network or a pool and attach it to the server
+        Get a floating IP by id.
+        @param floating_ip_id:
+        @param filters:
+        @return:
+        """
+        return self.conn.get_floating_ip(floating_ip_id, filters)
+
+    def add_ip_list(self, *, server, ips, wait=False, timeout=60, fixed_address=None, nat_destination=None):
+        """
+        Add ip list to server.
+        :param server:
+        :param ips:
+        :param wait:
+        :param timeout:
+        :param fixed_address:
+        :param nat_destination:
+        :return:
+        """
+        return self.conn.add_ip_list(server, ips, wait=wait, timeout=timeout, fixed_address=fixed_address,
+                                     nat_destination=nat_destination)
+
+    def create_floating_ip(self, *, network=None, server=None, fixed_address=None, nat_destination=None, port=None,
+                           wait=True, timeout=60):
+        """
+
         @param network:
         @param server:
+        @param fixed_address:
+        @param nat_destination:
+        @param port:
+        @param wait:
+        @param timeout:
         @return:
         """
-        floating_ip = self.conn.available_floating_ip(network=network, server=server)
-        self.conn.add_ip_list(server, [floating_ip["floating_ip_address"]])
-        return floating_ip
+        return self.conn.create_floating_ip(network=network, server=server, fixed_address=fixed_address,
+                                            nat_destination=nat_destination,
+                                            port=port, wait=wait, timeout=timeout)
 
     def get_images(self):
         """