Merge branch 'main' into improve_params

Author: scoopex

README.md

@@ -1,7 +1,12 @@
 ---
 admin_domain_password: "yolobanana"
 admin_vm_ssh_keypair_name: "my_ssh_public_key"
-admin_vm_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACLmNpHitBkZGVbWAFxZjUATNvLjSktAKwokFIQ9Z1k [email protected]"
+admin_vm_ssh_key: |
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACLmNpHitBkZGVbWAFxZjUATNvLjSktAKwokFIQ9Z1k [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDyw2z/C+5YugYNXQXbeop0AcOjmWZCvcmci/vOAboO8 [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHsMKOr3TEolg4+4hny/zBe4kLcjzkm+vkc932498fVD [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILa+/eL5ZM3AWKgm1h4/EFU9hOaSKqaoldHmNeg0qG46 [email protected]
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2wE2xiuO+i3qmDvu8kCCKX7U7H1diOICbWmR2UrKIxqWgcfWMQsT3WclotJKuVJuKIWyFD6ZNwwLuvC3RxVSqhCiWjqxg3jzJBj7/C1O3IYyLGTUl/x7Cky530lf/aj4wrwt3Ketk/4QNwgget2nCvOy0S2NDCJ3rL6oIUjdJekvRrFf9IbWeX8fqYYCoh1cYJWto1XYPnhMDAB/lqtjN1ssurLSKoJg/bUT7q/KkIvvA/BOR2NMqS0aGx+bKhdkeB22V/t75Ct8ymoCYk9+MTC9i/QX20Fi7835/W7Gl18J8NiO9ebaWyYbsxZ5klWXQa5EiLLBDZ82OR88G+0FjXp1Z3VG6FcpdYpW7sxrT21HEvWOnQACZCdlzwyBJ31id/LjDRhJU6BmZm0Sa9EOJNL8XVOUUzuoa0XL1mIVTsmLpUwqLSfw6Ditb+q4afFi0iYMe3JKOt+JmftvBgeQCjNUsCzk+Ny2j6dZKv2aeF5LOQZGRM3HzG39Gkir3q1zdWmCl4lc3QQBfr5ZcdAp+wQMFSgJAudKffO9kdDVNoyjgih7rD3E+JjJdhY9//WQEEBm2vfEqm7qqEQUAELd0JBCivJmOhUVH0rGbTrnkTBtLR4Au40W5aYaNQJ7+U3hTRrvpycSC1pUU3Wq3OXJd2FRDgKQJljQcpBw4V9j8GQ== Operator
 admin_vm_password: "yolobanana"
 vm_flavor: "SCS-1L-1"
 vm_image: "Ubuntu 24.04"

profiles/default.yaml

@@ -2,6 +2,10 @@
 admin_domain_password: "yolobanana"
 admin_vm_ssh_key: |
   ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACLmNpHitBkZGVbWAFxZjUATNvLjSktAKwokFIQ9Z1k [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDyw2z/C+5YugYNXQXbeop0AcOjmWZCvcmci/vOAboO8 [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHsMKOr3TEolg4+4hny/zBe4kLcjzkm+vkc932498fVD [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILa+/eL5ZM3AWKgm1h4/EFU9hOaSKqaoldHmNeg0qG46 [email protected]
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2wE2xiuO+i3qmDvu8kCCKX7U7H1diOICbWmR2UrKIxqWgcfWMQsT3WclotJKuVJuKIWyFD6ZNwwLuvC3RxVSqhCiWjqxg3jzJBj7/C1O3IYyLGTUl/x7Cky530lf/aj4wrwt3Ketk/4QNwgget2nCvOy0S2NDCJ3rL6oIUjdJekvRrFf9IbWeX8fqYYCoh1cYJWto1XYPnhMDAB/lqtjN1ssurLSKoJg/bUT7q/KkIvvA/BOR2NMqS0aGx+bKhdkeB22V/t75Ct8ymoCYk9+MTC9i/QX20Fi7835/W7Gl18J8NiO9ebaWyYbsxZ5klWXQa5EiLLBDZ82OR88G+0FjXp1Z3VG6FcpdYpW7sxrT21HEvWOnQACZCdlzwyBJ31id/LjDRhJU6BmZm0Sa9EOJNL8XVOUUzuoa0XL1mIVTsmLpUwqLSfw6Ditb+q4afFi0iYMe3JKOt+JmftvBgeQCjNUsCzk+Ny2j6dZKv2aeF5LOQZGRM3HzG39Gkir3q1zdWmCl4lc3QQBfr5ZcdAp+wQMFSgJAudKffO9kdDVNoyjgih7rD3E+JjJdhY9//WQEEBm2vfEqm7qqEQUAELd0JBCivJmOhUVH0rGbTrnkTBtLR4Au40W5aYaNQJ7+U3hTRrvpycSC1pUU3Wq3OXJd2FRDgKQJljQcpBw4V9j8GQ== Operator
 admin_vm_password: "yolobanana"
 vm_flavor: "SCS-2V-4"
 vm_image: "Ubuntu 24.04"

profiles/smoketest.yaml

@@ -2,6 +2,10 @@
 admin_domain_password: "yolobanana"
 admin_vm_ssh_key: |
   ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACLmNpHitBkZGVbWAFxZjUATNvLjSktAKwokFIQ9Z1k [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDyw2z/C+5YugYNXQXbeop0AcOjmWZCvcmci/vOAboO8 [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHsMKOr3TEolg4+4hny/zBe4kLcjzkm+vkc932498fVD [email protected]
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILa+/eL5ZM3AWKgm1h4/EFU9hOaSKqaoldHmNeg0qG46 [email protected]
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2wE2xiuO+i3qmDvu8kCCKX7U7H1diOICbWmR2UrKIxqWgcfWMQsT3WclotJKuVJuKIWyFD6ZNwwLuvC3RxVSqhCiWjqxg3jzJBj7/C1O3IYyLGTUl/x7Cky530lf/aj4wrwt3Ketk/4QNwgget2nCvOy0S2NDCJ3rL6oIUjdJekvRrFf9IbWeX8fqYYCoh1cYJWto1XYPnhMDAB/lqtjN1ssurLSKoJg/bUT7q/KkIvvA/BOR2NMqS0aGx+bKhdkeB22V/t75Ct8ymoCYk9+MTC9i/QX20Fi7835/W7Gl18J8NiO9ebaWyYbsxZ5klWXQa5EiLLBDZ82OR88G+0FjXp1Z3VG6FcpdYpW7sxrT21HEvWOnQACZCdlzwyBJ31id/LjDRhJU6BmZm0Sa9EOJNL8XVOUUzuoa0XL1mIVTsmLpUwqLSfw6Ditb+q4afFi0iYMe3JKOt+JmftvBgeQCjNUsCzk+Ny2j6dZKv2aeF5LOQZGRM3HzG39Gkir3q1zdWmCl4lc3QQBfr5ZcdAp+wQMFSgJAudKffO9kdDVNoyjgih7rD3E+JjJdhY9//WQEEBm2vfEqm7qqEQUAELd0JBCivJmOhUVH0rGbTrnkTBtLR4Au40W5aYaNQJ7+U3hTRrvpycSC1pUU3Wq3OXJd2FRDgKQJljQcpBw4V9j8GQ== Operator
 admin_vm_password: "yolobanana"
 vm_flavor: "SCS-2V-4"
 vm_image: "Ubuntu 24.04"

profiles/stresstest.yaml

@@ -44,6 +44,10 @@
                     help="Dump the created servers as an ansible inventory to the specified directory, "
                          "adds a ssh proxy jump for the hosts without a floating ip")
 
+parser.add_argument('--wait_for_machines', action="store_true",
+                    help="Wait for every machine to be created "
+                         "(normally the provisioning only waits for machines which use floating ips)")
+
 parser.add_argument('--config', type=str,
                     default="default.yaml",
                     help='The config file for environment creation, define a path to the'
@@ -113,7 +117,7 @@ def establish_connection():
         for workload_domain in workload_domains.values():
             for workload_project in workload_domain.get_projects(args.create_projects):
                 if args.create_machines:
-                    workload_project.get_and_create_machines(args.create_machines)
+                    workload_project.get_and_create_machines(args.create_machines, args.wait_for_machines)
                     if args.ansible_inventory:
                         workload_project.dump_inventory_hosts(args.ansible_inventory)
                 elif args.delete_machines:

src/openstack_workload_generator/__main__.py

@@ -98,6 +98,6 @@ def create_and_get_projects(self, create_projects: list[str]):
             self.workload_projects[project_name] = project
             project.close_connection()
 
-    def create_and_get_machines(self, machines: list[str]):
+    def create_and_get_machines(self, machines: list[str], wait_for_machines: bool):
         for project in self.workload_projects.values():
-            project.get_and_create_machines(machines)
+            project.get_and_create_machines(machines, wait_for_machines)

src/openstack_workload_generator/entities/domain.py

@@ -2,7 +2,6 @@
 import logging
 import os
 import sys
-from pprint import pformat
 from typing import Tuple
 import coloredlogs
 
@@ -23,6 +22,7 @@ class Config:
             'admin_vm_ssh_keypair_name': 'my_ssh_public_key',
             'project_ipv4_subnet': '192.168.200.0/24',
             'public_network': "public",
+            'network_mtu': '1500',
             'number_of_floating_ips_per_project': "1",
             'vm_flavor': 'SCS-1L-1',
             'vm_image': 'Ubuntu 24.04',
@@ -90,15 +90,17 @@ def check_config(cls):
         for quota_type in ["compute_quotas", "block_storage_quotas", "network_quotas"]:
             if quota_type not in Config._config:
                 continue
-            for key_name, value in Config._config[quota_type].keys():
-                Config.quota(key_name, quota_type, 1)
+            for key_name in Config._config[quota_type].keys():
+                Config.quota(key_name, quota_type, 0)
 
     @staticmethod
     def show_effective_config():
         Config.check_config(Config)
         LOGGER.info(
-            "The effective configuration from %s : \n>>>\n%s\n<<<" % (
-                Config._file, pformat(Config._config, indent=2, compact=False))
+            "The effective configuration from %s : \n>>>\n---\n%s\n<<<" % (
+                Config._file,
+                yaml.dump(Config._config, default_flow_style=False, width=10000),
+            )
         )
 
     @staticmethod
@@ -169,6 +171,10 @@ def quota(quota_name: str, quota_category: str, default_value: int) -> int:
         else:
             return default_value
 
+    @staticmethod
+    def get_network_mtu():
+        return int(Config.get("network_mtu", regex=r"\d+"))
+
 
 class DomainCache:
     _domains: dict[str, str] = dict()

src/openstack_workload_generator/entities/helpers.py

@@ -53,7 +53,7 @@ def wait_for_delete(self):
         self.conn.compute.wait_for_delete(self.obj)
         LOGGER.warning(f"Machine {self.machine_name} in {self.obj.project_id} is deleted now")
 
-    def create_or_get_server(self, network: Network):
+    def create_or_get_server(self, network: Network, wait_for_machine: bool):
 
         if self.obj:
             LOGGER.info(
@@ -82,6 +82,8 @@ def create_or_get_server(self, network: Network):
             ],
             key_name=Config.get_admin_vm_ssh_keypair_name(),
         )
+        if wait_for_machine:
+            self.wait_for_server()
         if self.obj:
             LOGGER.info(f"Created server {self.obj.name}/{self.obj.id} in {ProjectCache.ident_by_id(network.project_id)}")
         else:

src/openstack_workload_generator/entities/machine.py

@@ -83,9 +83,9 @@ def create_and_get_network_setup(self) -> Network:
         return network
 
     def create_and_get_router(self, subnet: Subnet) -> Router | None:
-        public_network = self.conn.network.find_network('public')
+        public_network = self.conn.network.find_network(Config.get_public_network())
         if not public_network:
-            LOGGER.error("There is no 'public' network")
+            LOGGER.error(f"There is no '{Config.get_public_network()}' network, not adding floating ips")
             return None
 
         if self.obj_router:
@@ -115,7 +115,7 @@ def create_and_get_network(self) -> Network:
         self.obj_network = self.conn.network.create_network(
             name=self.network_name,
             project_id=self.project.id,
-            mtu=1342
+            mtu=Config.get_network_mtu(),
         )
         if not self.obj_network:
             raise RuntimeError(f"Unable to create network {self.network_name}")

src/openstack_workload_generator/entities/network.py

@@ -215,7 +215,7 @@ def delete_project(self):
             self._admin_conn.network.delete_security_group(sg.id)
         ##########################################################################################
 
-    def get_and_create_machines(self, machines: list[str]):
+    def get_and_create_machines(self, machines: list[str], wait_for_machines: bool):
         if "none" in machines:
             LOGGER.warning("Not creating a virtual machine, because 'none' was in the list")
             self.close_connection()
@@ -231,7 +231,7 @@ def get_and_create_machines(self, machines: list[str]):
                 if self.workload_network is None or self.workload_network.obj_network is None:
                     raise RuntimeError("No Workload network object")
 
-                machine.create_or_get_server(self.workload_network.obj_network)
+                machine.create_or_get_server(self.workload_network.obj_network, wait_for_machines)
 
                 if machine.floating_ip:
                     self.ssh_proxy_jump = machine.floating_ip
@@ -251,21 +251,28 @@ def dump_inventory_hosts(self, directory_location: str):
                 raise RuntimeError(f"Invalid reference to server for {workload_machine.machine_name}")
 
             workload_machine.update_assigned_ips()
-            data = {
-                "id": workload_machine.obj.id,
-                "status": workload_machine.obj.status,
-                "hypervisor": workload_machine.obj['OS-EXT-SRV-ATTR:hypervisor_hostname'],
+
+            if not workload_machine.internal_ip:
+                raise RuntimeError(f"Unable to get associated ip address for {workload_machine.machine_name}")
+
+            data: dict[str, str | dict[str, str]] = {
+                "openstack": {
+                    "machine_id": workload_machine.obj.id,
+                    "machine_status": workload_machine.obj.status,
+                    "hypervisor": workload_machine.obj['OS-EXT-SRV-ATTR:hypervisor_hostname'],
+                    "domain": self.domain.name,
+                    "project": workload_machine.project.name,
+                },
                 "hostname": workload_machine.machine_name,
-                "project": workload_machine.project.name,
-                "domain": self.domain.name,
                 "ansible_host": workload_machine.floating_ip or workload_machine.internal_ip,
                 "internal_ip": workload_machine.internal_ip,
             }
 
             if self.ssh_proxy_jump and not workload_machine.floating_ip:
                 data["ansible_ssh_common_args"] = f"-o ProxyJump={self.ssh_proxy_jump} "
 
-            base_dir = f"{directory_location}/{data['domain']}-{data['project']}-{data['hostname']}"
+            base_dir = f"{directory_location}/{self.domain.name}-{workload_machine.project.name}-{workload_machine.machine_name}"
+
             filename = f'{base_dir}/data.yml'
             os.makedirs(base_dir, exist_ok=True)
             with open(filename, 'w') as file: