Better clouds yaml, get passwords interactively, bugfix quota (#31)

* improve clouds yaml generation
* Improve cloud yaml generation
* improve gitignore
* Set quotas on every execution and bugfix the lookup
* Add facility to ask passwords
* add documentation

---------

Signed-off-by: Marc Schöchlin <[email protected]>
Co-authored-by: Marc Schöchlin <[email protected]>

Author: scoopex

.gitignore

@@ -2,4 +2,4 @@ venv
 __pycache__
 clouds*.yaml
 *clouds.yaml
-
+*.yaml_????-??-??_??-??-??

README.md

@@ -26,50 +26,70 @@ basis for later automation.
 # Usage
 
 ```
-$ ./openstack_workload_generator --help
-usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD]
-                                                   [--ansible_inventory [ANSIBLE_INVENTORY]]
-                                                   [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines]
-                                                   [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]]
-                                                   [--config CONFIG]
-                                                   (--create_domains DOMAINNAME [DOMAINNAME ...] |
-                                                   --delete_domains DOMAINNAME [DOMAINNAME ...])
-                                                   [--create_projects PROJECTNAME [PROJECTNAME ...] |
-                                                   --delete_projects PROJECTNAME [PROJECTNAME ...]]
-                                                   [--create_machines SERVERNAME [SERVERNAME ...] |
-                                                   --delete_machines SERVERNAME [SERVERNAME ...]]
+$ openstack_workload_generator --help
+usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD] [--ansible_inventory [ANSIBLE_INVENTORY]]
+                                                   [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines] [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]]
+                                                   [--config CONFIG] (--create_domains DOMAINNAME [DOMAINNAME ...] | --delete_domains DOMAINNAME [DOMAINNAME ...])
+                                                   [--create_projects PROJECTNAME [PROJECTNAME ...] | --delete_projects PROJECTNAME [PROJECTNAME ...]]
+                                                   [--create_machines SERVERNAME [SERVERNAME ...] | --delete_machines SERVERNAME [SERVERNAME ...]]
 
 options:
   -h, --help            show this help message and exit
   --log_level loglevel  The loglevel
-  --os_cloud OS_CLOUD   The openstack config to use, defaults to the value of the OS_CLOUD environment variable or
-                        "admin" if the variable is not set
+  --os_cloud OS_CLOUD   The openstack config to use, defaults to the value of the OS_CLOUD environment variable or "admin" if the variable is not set
   --ansible_inventory [ANSIBLE_INVENTORY]
-                        Dump the created servers as an ansible inventory to the specified directory, adds a ssh
-                        proxy jump for the hosts without a floating ip
+                        Dump the created servers as an ansible inventory to the specified directory, adds a ssh proxy jump for the hosts without a floating ip
   --clouds_yaml [CLOUDS_YAML]
                         Use a specific clouds.yaml file
-  --wait_for_machines   Wait for every machine to be created (normally the provisioning only waits for machines
-                        which use floating ips)
+  --wait_for_machines   Wait for every machine to be created (normally the provisioning only waits for machines which use floating ips)
   --generate_clouds_yaml [GENERATE_CLOUDS_YAML]
                         Generate a openstack clouds.yaml file
-  --config CONFIG       The config file for environment creation, define a path to the yaml file or a subpath in
-                        the profiles folder
+  --config CONFIG       The config file for environment creation, define a path to the yaml file or a subpath in the profiles folder of the tool (you can overload
+                        the search path by setting the OPENSTACK_WORKLOAD_MANAGER_PROFILES environment variable)
   --create_domains DOMAINNAME [DOMAINNAME ...]
                         A list of domains to be created
   --delete_domains DOMAINNAME [DOMAINNAME ...]
                         A list of domains to be deleted, all child elements are recursively deleted
   --create_projects PROJECTNAME [PROJECTNAME ...]
                         A list of projects to be created in the created domains
   --delete_projects PROJECTNAME [PROJECTNAME ...]
-                        A list of projects to be deleted in the created domains, all child elements are
-                        recursively deleted
+                        A list of projects to be deleted in the created domains, all child elements are recursively deleted
   --create_machines SERVERNAME [SERVERNAME ...]
                         A list of vms to be created in the created domains
   --delete_machines SERVERNAME [SERVERNAME ...]
                         A list of vms to be deleted in the created projects
 ```
 
+# Configuration
+
+The following cnfigurations:
+
+* `admin_domain_password`
+  * the password for the domain users which are created (User `<domain-name>_admin`)
+  * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way
+* `admin_vm_password`:
+  * the password for the operating system user (the username depends on the type of image you are using)
+  * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way
+* `vm_flavor`:
+  * the name of the flavor used to create virtual machines
+  * see `openstack flavor list`
+* `vm_image`:
+  * the name of the image used to create virtual machines
+  * see `openstack image list`
+* `vm_volume_size_gb`:
+  * the size of the persistent root volume
+* `project_ipv4_subnet`:
+  * the network cidr of the internal network
+* `*_quotas`:
+  * the quotas for the created projects
+  * execute the tool with `--log_level DEBUG` to see the configurable values
+* `public_network`:
+  * The name of the public network which is used for floating ips
+* `admin_vm_ssh_key`:
+  * A multiline string which ssh public keys
+
+```
+
 # Testing Scenarios
 
 ## Example usage: A minimal scenario

src/openstack_workload_generator/__main__.py

@@ -22,6 +22,52 @@
 )
 
 
+def establish_connection():
+    if args.clouds_yaml is None:
+        config = loader.OpenStackConfig()
+    else:
+        LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}")
+        config = loader.OpenStackConfig(config_files=[args.clouds_yaml])
+    cloud_config = config.get_one(args.os_cloud)
+    return Connection(config=cloud_config)
+
+
+def generated_clouds_yaml():
+    LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}")
+    clouds_yaml_data_new = {"clouds": clouds_yaml_data}
+    initial_entries = 0
+    generated_entries = 0
+    total_entries = 0
+    if os.path.exists(args.generate_clouds_yaml):
+        with open(args.generate_clouds_yaml, "r") as file:
+            existing_data = yaml.safe_load(file)
+
+        initial_entries = len(existing_data.get("clouds", []))
+        backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}"
+        logging.warning(
+            f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values"
+        )
+        shutil.copy2(
+            args.generate_clouds_yaml,
+            f"{args.generate_clouds_yaml}_{iso_timestamp()}",
+        )
+
+        generated_entries = len(clouds_yaml_data_new.get("clouds", []))
+        clouds_yaml_data_new = deep_merge_dict(existing_data, clouds_yaml_data_new)
+        total_entries = len(clouds_yaml_data_new.get("clouds", []))
+    with open(args.generate_clouds_yaml, "w") as file:
+        yaml.dump(
+            clouds_yaml_data_new,
+            file,
+            default_flow_style=False,
+            explicit_start=True,
+        )
+    LOGGER.info(
+        f"Generated {generated_entries} entries, number of entries in "
+        f"{args.generate_clouds_yaml} is now {total_entries} (old {initial_entries} entries)"
+    )
+
+
 LOGGER = logging.getLogger()
 
 parser = argparse.ArgumentParser(prog="Create workloads on openstack installations")
@@ -141,22 +187,12 @@
 
 setup_logging(args.log_level)
 
-
-def establish_connection():
-    if args.clouds_yaml is None:
-        config = loader.OpenStackConfig()
-    else:
-        LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}")
-        config = loader.OpenStackConfig(config_files=[args.clouds_yaml])
-    cloud_config = config.get_one(args.os_cloud)
-    return Connection(config=cloud_config)
-
-
 time_start = time.time()
 
 Config.load_config(args.config)
 Config.show_effective_config()
 
+
 if args.create_domains:
     conn = establish_connection()
     workload_domains: dict[str, WorkloadGeneratorDomain] = dict()
@@ -172,48 +208,27 @@ def establish_connection():
 
         for workload_domain in workload_domains.values():
             for workload_project in workload_domain.get_projects(args.create_projects):
+                if args.generate_clouds_yaml:
+                    clouds_yaml_data[
+                        f"{workload_domain.domain_name}-{workload_project.project_name}"
+                    ] = workload_project.get_clouds_yaml_data()
+
                 if args.create_machines:
                     workload_project.get_and_create_machines(
                         args.create_machines, args.wait_for_machines
                     )
                     if args.ansible_inventory:
                         workload_project.dump_inventory_hosts(args.ansible_inventory)
-                    if args.generate_clouds_yaml:
-                        clouds_yaml_data[
-                            f"{workload_domain.domain_name}-{workload_project.project_name}"
-                        ] = workload_project.get_clouds_yaml_data()
+
                 elif args.delete_machines:
                     for machine_obj in workload_project.get_machines(
                         args.delete_machines
                     ):
                         machine_obj.delete_machine()
 
         if args.generate_clouds_yaml:
-            LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}")
-            clouds_yaml_data_new = {"clouds": clouds_yaml_data}
-
-            if os.path.exists(args.generate_clouds_yaml):
-                with open(args.generate_clouds_yaml, "r") as file:
-                    existing_data = yaml.safe_load(file)
-                backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}"
-                logging.warning(
-                    f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values"
-                )
-                shutil.copy2(
-                    args.generate_clouds_yaml,
-                    f"{args.generate_clouds_yaml}_{iso_timestamp()}",
-                )
-                clouds_yaml_data_new = deep_merge_dict(
-                    existing_data, clouds_yaml_data_new
-                )
-
-            with open(args.generate_clouds_yaml, "w") as file:
-                yaml.dump(
-                    clouds_yaml_data_new,
-                    file,
-                    default_flow_style=False,
-                    explicit_start=True,
-                )
+            generated_clouds_yaml()
+
         sys.exit(0)
     elif args.delete_projects:
         conn = establish_connection()

src/openstack_workload_generator/entities/domain.py

@@ -97,6 +97,7 @@ def create_and_get_projects(self, create_projects: list[str]):
 
         for project_name in create_projects:
             if project_name in self.workload_projects:
+                self.workload_projects[project_name].adapt_quota()
                 continue
             project = WorkloadGeneratorProject(
                 self.conn, project_name, self.obj, self.workload_user

src/openstack_workload_generator/entities/helpers.py

@@ -1,3 +1,4 @@
+import getpass
 import inspect
 import logging
 import os
@@ -62,9 +63,14 @@ def get(key: str, regex: str = ".+", multi_line: bool = False) -> str:
 
     @staticmethod
     def load_config(config_file: str):
-        potential_profile_file = str(
+
+        profile_path = str(
             os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
-            + f"/../../../profiles/{config_file}"
+            + "/../../../profiles/"
+        )
+        potential_profile_file = str(
+            Path(os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", profile_path))
+            / Path(config_file)
         )
 
         if os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", None):
@@ -135,7 +141,9 @@ def get_number_of_floating_ips_per_project() -> int:
 
     @staticmethod
     def get_admin_vm_password() -> str:
-        return Config.get("admin_vm_password")
+        if Config.get("admin_vm_password").upper() == "ASK_PASSWORD":
+            Config._config["admin_vm_password"] = getpass.getpass("Enter the wanted admin_vm_password: ")
+        return Config.get("admin_vm_password",  regex=r".{5,}")
 
     @staticmethod
     def get_vm_flavor() -> str:
@@ -171,6 +179,8 @@ def get_admin_vm_ssh_key() -> str:
 
     @staticmethod
     def get_admin_domain_password() -> str:
+        if Config.get("admin_domain_password").upper() == "ASK_PASSWORD":
+            Config._config["admin_domain_password"] = getpass.getpass("Enter the wanted admin_domain_password: ")
         return Config.get("admin_domain_password", regex=r".{5,}")
 
     @staticmethod
@@ -192,7 +202,9 @@ def configured_quota_names(quota_category: str) -> list[str]:
     @staticmethod
     def quota(quota_name: str, quota_category: str, default_value: int) -> int:
         if quota_category in Config._config:
-            value = Config._config.get(quota_name, default_value)
+            value = Config._config[quota_category].get(  # type: ignore
+                quota_name, default_value
+            )
             if isinstance(value, int):
                 return value
             else:
@@ -246,7 +258,7 @@ def setup_logging(log_level: str) -> Tuple[logging.Logger, str]:
     )
     logger = logging.getLogger()
     log_file = "STDOUT"
-    logging.basicConfig(format=log_format_string, level=log_level)
+    logging.basicConfig(format=log_format_string, level=log_level.upper())
 
     coloredlogs.DEFAULT_FIELD_STYLES["levelname"] = {"bold": True, "color": ""}
     coloredlogs.install(fmt=log_format_string, level=log_level.upper())

src/openstack_workload_generator/entities/project.py

@@ -169,8 +169,6 @@ def _set_quota(self, quota_category: str):
         else:
             raise RuntimeError(f"Not implemented: {quota_category}")
 
-        # service_obj = getattr(self._admin_conn, api_area)
-        # current_quota = service_obj.get_quota_set(self.obj.id)
         LOGGER.debug(f"current quotas for {quota_category} : {current_quota}")
 
         new_quota = {}
@@ -193,7 +191,7 @@ def _set_quota(self, quota_category: str):
                 )
                 new_quota[key_name] = new_value
 
-        if len(new_quota):
+        if len(new_quota.keys()) > 0:
             set_quota_method = getattr(self._admin_conn, f"set_{api_area}_quotas")
             set_quota_method(self.obj.id, **new_quota)
             LOGGER.info(