Enhance chargeback synthetic script and temple

- Used Gemini and Cursor
- add scenario(parameter) files.
    - Each scenrio file defines one test scenario with specific metrics
    - Scenario file names begin with "test_"
    - All scenerio files are found and read automatically
    - Output files have same name as scenario but different extension
    - Price and qty variable are not dynamic => "change over time"
- Closes https://issues.redhat.com/browse/OSPRH-24867
- Closes https://issues.redhat.com/browse/OSPRH-24865

Author: ayefimov-1

roles/telemetry_chargeback/README.md

@@ -18,12 +18,24 @@ It is expected to be run **after** a successful deployment and configuration of
 
 Role Variables
 --------------
-The role uses a few primary variables to control the testing environment and execution.
+The role uses the following variables to control the testing environment and execution.
+
+### User-Configurable Variables (defaults/main.yml)
 
 | Variable | Default Value | Description |
 |----------|---------------|-------------|
 | `openstack_cmd` | `openstack` | The command used to execute OpenStack CLI calls. This can be customized if the binary is not in the standard PATH. |
 
+### Internal Variables (vars/main.yml)
+
+| Variable | Default Value | Description |
+|----------|---------------|-------------|
+| `logs_dir_zuul` | `/home/zuul/ci-framework-data/logs` | Remote directory for log files. |
+| `artifacts_dir_zuul` | `/home/zuul/ci-framework-data/artifacts` | Remote directory for generated artifacts. |
+| `ck_scenario_dir` | `{{ role_path }}/files` | Local directory containing scenario files (test_*.yml). |
+| `ck_synth_script` | `{{ role_path }}/files/gen_synth_loki_data.py` | Path to the Python script that generates synthetic Loki data. |
+| `ck_data_template` | `{{ role_path }}/template/loki_data_templ.j2` | Path to the Jinja2 template for Loki data format. |
+
 Dependencies
 ------------
 This role has no direct hard dependencies on other Ansible roles.

roles/telemetry_chargeback/defaults/main.yml

@@ -1,10 +1,2 @@
 ---
 openstack_cmd: "openstack"
-
-ck_synth_script: "{{ role_path }}/files/gen_synth_loki_data.py"
-ck_data_template: "{{ role_path }}/files/loki_data_templ.j2"
-ck_data_config: "{{ role_path }}/files/test_static.yml"
-ck_output_file_local: "{{ role_path }}/files/loki_synth_data.json"
-
-# Output directory for test artifacts
-logs_dir: "/home/zuul/ci-framework-data/tests/feature-verification-tests" 

roles/telemetry_chargeback/files/gen_synth_loki_data.py

@@ -5,10 +5,44 @@
 import yaml
 from datetime import datetime, timezone, timedelta
 from pathlib import Path
-from typing import Dict, Any
+from typing import Dict, Any, List, Union
 from jinja2 import Environment
 
 
+def _get_value_for_step(
+    values: List[Union[int, float]],
+    step_idx: int,
+    num_steps: int
+) -> Union[int, float]:
+    """
+    Get the appropriate value from a list based on the current step index.
+
+    Values are distributed evenly across all steps. For example, if there are
+    12 steps and 4 values, each value covers 3 steps:
+    - Steps 0-2: values[0]
+    - Steps 3-5: values[1]
+    - Steps 6-8: values[2]
+    - Steps 9-11: values[3]
+
+    Args:
+        values: List of values to choose from.
+        step_idx: Current step index (0-based).
+        num_steps: Total number of steps.
+
+    Returns:
+        The value corresponding to the current step.
+    """
+    num_values = len(values)
+    if num_values == 1:
+        return values[0]
+
+    # Calculate how many steps each value covers
+    steps_per_value = num_steps / num_values
+    # Determine which value index to use, clamping to valid range
+    value_idx = min(int(step_idx // steps_per_value), num_values - 1)
+    return values[value_idx]
+
+
 # --- Configure logging with a default level that can be changed ---
 logging.basicConfig(
     level=logging.INFO,
@@ -86,9 +120,8 @@ def generate_loki_data(
         time_step_seconds (int): The duration of each log entry in seconds.
         config (Dict[str, Any]): Configuration dictionary loaded from file.
     """
-    # Get constants from config
-    constants = config.get("constants", {})
-    invalid_timestamp = constants.get("invalid_timestamp", "INVALID_TIMESTAMP")
+    # Hardcoded constant for invalid timestamp display
+    invalid_timestamp = "INVALID_TIMESTAMP"
 
     # --- Step 1: Generate the data structure first ---
     logger.info(
@@ -201,12 +234,18 @@ def generate_loki_data(
                 f"groupby must be a dictionary for {log_type_name}"
             )
 
+        # Ensure qty and price are lists for step-based distribution
+        qty_val = log_type_config["qty"]
+        price_val = log_type_config["price"]
+        qty_list = qty_val if isinstance(qty_val, list) else [qty_val]
+        price_list = price_val if isinstance(price_val, list) else [price_val]
+
         log_types[log_type_name] = {
             "type": log_type_config["type"],
             "unit": log_type_config["unit"],
             "description": log_type_config.get("description"),
-            "qty": log_type_config["qty"],
-            "price": log_type_config["price"],
+            "qty": qty_list,
+            "price": price_list,
             "groupby": groupby.copy(),
             "metadata": log_type_config.get("metadata", {})
         }
@@ -232,6 +271,10 @@ def tojson_preserve_order(obj):
     # --- Render the template in one pass with all the data ---
     logger.info("Rendering final output...")
 
+    # Calculate total number of steps for value distribution
+    num_steps = len(log_data_list)
+    logger.debug(f"Total number of time steps: {num_steps}")
+
     # Pre-calculate log types with date fields for each time step
     log_types_list = []
     for idx, item in enumerate(log_data_list):
@@ -268,6 +311,13 @@ def tojson_preserve_order(obj):
             log_type_with_dates = log_type_data.copy()
             log_type_with_dates["groupby"] = log_type_data["groupby"].copy()
             log_type_with_dates["groupby"].update(date_fields)
+            # Select qty and price based on step index distribution
+            log_type_with_dates["qty"] = _get_value_for_step(
+                log_type_data["qty"], idx, num_steps
+            )
+            log_type_with_dates["price"] = _get_value_for_step(
+                log_type_data["price"], idx, num_steps
+            )
             log_types_with_dates[log_type_name] = log_type_with_dates
 
         log_types_list.append(log_types_with_dates)

roles/telemetry_chargeback/files/test_all_qty_zero.yml

@@ -0,0 +1,136 @@
+# Scenario configuration for synthetic Loki log data generation
+
+# Time range configuration
+generation:
+  days: 30
+  step_seconds: 3600
+
+# Log type definitions
+log_types:
+  - name: ceilometer_cpu
+    type: instance
+    unit: instance
+    description: "max number of cpus used in time step"
+    qty:
+      - 0.0 
+    price:
+      - 0.3 
+    groupby:
+      id: null
+      user_id: null
+      project_id: null
+    metadata:
+      flavor_name: null
+      flavor_id: "" 
+      vcpus: ""
+
+  - name: ceilometer_image_size
+    type: ceilometer_image_size
+    unit: MiB
+    description: "Size of ceilometer image"
+    qty:
+      - 0.0
+    price:
+      - 0.02
+    groupby:
+      id: null
+      user_id: null
+      project_id: null
+    metadata:
+      container_format: null
+      disk_format: null
+
+  - name: ceilometer_disk_ephemeral_size
+    type: null
+    unit: GiB
+    description: "Max at each timestep"
+    qty: 
+      - 0.0
+    price:
+      - 0.1
+    groupby:
+      id: null
+      user_id: null
+      project_id: null
+    metadata:
+      type: null
+
+  - name: ceilometer_disk_root_size
+    type: null
+    unit: GiB
+    description: null
+    qty: 
+      - 0.0
+    price: 
+      - 1.00
+    groupby:
+      id: null
+      user_id: null
+      project_id: null    
+    metadata:
+      disk_format: null
+
+  - name: ceilometer_network_outgoing_bytes
+    type: null
+    unit: B
+    description: null
+    qty: 
+      - 0.0
+    price:
+      - 0.02    
+    groupby:
+      id: null
+      user_id: null
+      project_id: null
+    metadata:
+      vm_instance: null
+
+  - name: ceilometer_network_incoming_bytes
+    type: null
+    unit: B
+    description: null
+    qty:
+      - 0.0
+    price:
+      - 0.02    
+    groupby:
+      id: null
+      user_id: null
+      project_id: null    
+    metadata:
+      vm_instance: null
+
+  - name: ceilometer_ip_floating
+    type: null
+    unit: ip
+    description: null
+    qty: 
+      - 0.0
+    price: 
+      - 0.50
+    groupby:
+      id: null
+      user_id: null
+      project_id: null    
+    metadata:
+      state: null
+
+# Required fields for validation (top-level fields only, not nested in groupby)
+required_fields:
+  - type
+  - unit
+  - qty
+  - price
+  - groupby
+  - metadata
+
+# Date field names to add to groupby
+date_fields:
+  - week_of_the_year
+  - day_of_the_year
+  - month
+  - year
+
+# Loki stream configuration
+loki_stream:
+  service: cloudkitty

roles/telemetry_chargeback/files/test_dyn_basic.yml

@@ -0,0 +1,68 @@
+# Scenario configuration for synthetic Loki log data generation
+
+# Time range configuration
+generation:
+  days: 1
+  step_seconds: 7200
+
+# Log type definitions
+log_types:
+  - name: ceilometer_image_size
+    type: ceilometer_image_size
+    unit: MiB
+    description: null
+    qty:
+      - 20.6
+      - 25.0
+      - 26.0
+      - 30.5
+    price:
+      - 0.02
+      - 0.03
+      - 0.04
+      - 0.07
+      - 0.10
+    groupby:
+      id: cd65d30f-8b94-4fa3-95dc-e3b429f479b2
+      project_id: 0030775de80e4d84a4fd0d73e0a1b3a7
+      user_id: null
+    metadata:
+      container_format: bare
+      disk_format: qcow2
+
+  - name: instance
+    type: instance
+    unit: instance
+    description: null
+    qty:
+      - 1.0
+    price:
+      - 0.3
+    groupby:
+      id: de168c31-ed44-4a1a-a079-51bd238a91d6
+      project_id: 9cf5bcfc61a24682acc448af2d062ad2
+      user_id: c29ab6e886354bbd88ee9899e62d1d40
+    metadata:
+      flavor_name: m1.tiny
+      flavor_id: "1"
+      vcpus: ""
+
+# Required fields for validation (top-level fields only, not nested in groupby)
+required_fields:
+  - type
+  - unit
+  - qty
+  - price
+  - groupby
+  - metadata
+
+# Date field names to add to groupby
+date_fields:
+  - week_of_the_year
+  - day_of_the_year
+  - month
+  - year
+
+# Loki stream configuration
+loki_stream:
+  service: cloudkitty

roles/telemetry_chargeback/files/test_static.yml

@@ -2,17 +2,19 @@
 
 # Time range configuration
 generation:
-  days: 5
-  step_seconds: 300
+  days: 1
+  step_seconds: 3600
 
 # Log type definitions
 log_types:
   - name: ceilometer_image_size
     type: ceilometer_image_size
     unit: MiB
     description: null
-    qty: 20.6
-    price: 0.02
+    qty:
+      - 20.6
+    price:
+      - 0.02
     groupby:
       id: cd65d30f-8b94-4fa3-95dc-e3b429f479b2
       project_id: 0030775de80e4d84a4fd0d73e0a1b3a7
@@ -25,8 +27,10 @@ log_types:
     type: instance
     unit: instance
     description: null
-    qty: 1.0
-    price: 0.3
+    qty:
+      - 1.0
+    price:
+      - 0.3
     groupby:
       id: de168c31-ed44-4a1a-a079-51bd238a91d6
       project_id: 9cf5bcfc61a24682acc448af2d062ad2
@@ -55,7 +59,3 @@ date_fields:
 # Loki stream configuration
 loki_stream:
   service: cloudkitty
-
-# Error messages and constants
-constants:
-  invalid_timestamp: "INVALID_TIMESTAMP"