Adding debug feature

Author: ayefimov-1

roles/telemetry_chargeback/README.md

@@ -80,7 +80,7 @@ Loki push/query URLs are set dynamically in `setup_loki_env.yml` from the Cloudk
 | Option | Description |
 |--------|--------------|
 | `--tmpl` | Path to the Jinja2 template (e.g. `loki_data_templ.j2`). |
-| `-t`, `--test` | Path to the scenario YAML (e.g. `test_static_basic.yml`). |
+| `-t`, `--test` | Path to the scenario YAML (e.g. `test_dyn_basic.yml`). |
 | `-o`, `--output` | Path to the output JSON file. |
 | `-p`, `--project-id` | Optional; overrides `groupby.project_id` in every log entry. |
 | `-u`, `--user-id` | Optional; overrides `groupby.user_id` in every log entry. |
@@ -124,14 +124,9 @@ Each scenario file defines:
 * **date_fields** — Date field names to merge into groupby (week_of_the_year, day_of_the_year, month, year).
 * **loki_stream** — Loki stream configuration (service name).
 
-**groupby.id** should be consistent by metric type across all scenario files so that the same type always uses the same id. The reference mapping is defined in `_test_all_qty_zero.yml` (e.g. `ceilometer_cpu_num` → me1, `ceilometer_image_size` → me2, `ceilometer_ip_floating` → me7).
+**groupby.id** should be consistent by metric type across scenario files so that the same type always uses the same id.
 
-Example scenario files:
-
-* `test_static_basic.yml` — Basic static values for qty and price.
-* `test_static_basic_gid.yml` — Same as above with explicit groupby ids.
-* `test_dyn_basic.yml` — Dynamic values distributed across time steps.
-* `_test_all_qty_zero.yml` — Reference scenario (all quantities zero); defines the standard groupby.id mapping. Not auto-discovered.
+Scenario files matching `test_*.yml` in the `files/` directory are automatically discovered and processed. Files whose names start with an underscore are not auto-discovered.
 
 Dependencies
 ------------

roles/telemetry_chargeback/files/gen_db_summary.py

@@ -11,7 +11,9 @@
 from collections import Counter
 from pathlib import Path
 
-REQUIRED_KEYS = frozenset({"start", "end", "type", "unit", "qty", "price", "groupby"})
+REQUIRED_KEYS = frozenset(
+    {"start", "end", "type", "unit", "qty", "price", "groupby"}
+)
 
 
 def _is_valid_timestep(s: str) -> bool:
@@ -20,7 +22,7 @@ def _is_valid_timestep(s: str) -> bool:
 
 
 def _has_required_keys(obj: dict) -> bool:
-    """Return True if obj has all required keys (start, end, type, unit, qty, price, groupby)."""
+    """Return True if obj has all required keys."""
     return REQUIRED_KEYS.issubset(obj.keys())
 
 
@@ -99,7 +101,7 @@ def _find_arrays_in_text(text: str) -> list[tuple[str, str]]:
             if c == "]":
                 depth -= 1
                 if depth == 0:
-                    slice_str = text[start : j + 1]
+                    slice_str = text[start:j + 1]
                     try:
                         arr = json.loads(slice_str)
                         if (
@@ -110,7 +112,8 @@ def _find_arrays_in_text(text: str) -> list[tuple[str, str]]:
                             and isinstance(arr[1], str)
                         ):
                             entry = json.loads(arr[1])
-                            if isinstance(entry, dict) and _has_required_keys(entry):
+                            if (isinstance(entry, dict) and
+                                    _has_required_keys(entry)):
                                 pairs.append((arr[0], arr[1]))
                     except json.JSONDecodeError:
                         pass
@@ -150,7 +153,8 @@ def extract_and_sort(json_path: Path) -> list[tuple[str, str]]:
 
 def main() -> None:
     parser = argparse.ArgumentParser(
-        description="Extract and sort Loki log entries from JSON; print each and log_count."
+        description="Extract and sort Loki log entries from JSON; "
+        "print each and log_count."
     )
     parser.add_argument(
         "-j", "--json",
@@ -162,33 +166,38 @@ def main() -> None:
         "-o", "--output",
         type=Path,
         default=None,
-        help="Output path (default: same as input with _total.yml suffix, e.g. file.json -> file_total.yml).",
+        help="Output path (default: input with _total.yml suffix).",
     )
     parser.add_argument(
         "--debug",
         type=Path,
         default=None,
         metavar="DIR",
-        help="Debug directory: create if needed and write timestep list to <input_stem>_diff.txt.",
+        help="Debug dir: create if needed, write timestep list to "
+        "<input_stem>_diff.txt.",
     )
     args = parser.parse_args()
 
     if not args.json.exists():
         print(f"Error: input file not found: {args.json}", file=sys.stderr)
         sys.exit(1)
 
-    output_path = args.output if args.output is not None else (args.json.parent / f"{args.json.stem}_total.yml")
+    output_path = (
+        args.output if args.output is not None
+        else (args.json.parent / f"{args.json.stem}_total.yml")
+    )
     pairs = extract_and_sort(args.json)
     out = output_path.open("w", encoding="utf-8")
 
-    # Debug: write timestep list to <input_stem>_diff.txt in --debug directory (skip if "" or null)
+    # Debug: write timestep list to <input_stem>_diff.txt (skip if "" or null)
     _debug_dir = str(args.debug).strip() if args.debug is not None else ""
     if _debug_dir and _debug_dir != ".":
         args.debug.mkdir(parents=True, exist_ok=True)
         debug_path = args.debug / f"{args.json.stem}_diff.txt"
         with debug_path.open("w", encoding="utf-8") as dbg:
             for timestep, log_str in pairs:
-                print(json.dumps([timestep, log_str], ensure_ascii=False), file=dbg)
+                line = json.dumps([timestep, log_str], ensure_ascii=False)
+                print(line, file=dbg)
 
     # Count unique timesteps and entries per timestep
     log_count = len(pairs)
@@ -199,7 +208,7 @@ def main() -> None:
     if counts and not all(c == metrics_per_step for c in counts):
         metrics_per_step = "ERROR"
 
-    # Timesteps with error: those whose count differs from the expected (first) count
+    # Timesteps with error: count differs from expected (first) count
     expected_count = counts[0] if counts else 0
     timesteps_with_error = [
         ts for ts, cnt in counts_per_timestep.items()
@@ -217,27 +226,40 @@ def main() -> None:
         time_end_begin = last_entry.get("start")
         time_end = last_entry.get("end")
     else:
-        begin_nano = end_nano = time_begin = time_begin_end = time_end_begin = time_end = None
+        begin_nano = end_nano = time_begin = time_begin_end = None
+        time_end_begin = time_end = None
 
     try:
         print("---", file=out)
         print("time:", file=out)
         print("  begin_step:", file=out)
-        print(f"    nanosec: {begin_nano}" if begin_nano is not None else "    nanosec: null", file=out)
-        print(f"    begin: {repr(time_begin)}" if time_begin is not None else "    begin: null", file=out)
-        print(f"    end: {repr(time_begin_end)}" if time_begin_end is not None else "    end: null", file=out)
+        _bn = (f"    nanosec: {begin_nano}" if begin_nano
+               else "    nanosec: null")
+        print(_bn, file=out)
+        _b = (f"    begin: {repr(time_begin)}" if time_begin
+              else "    begin: null")
+        print(_b, file=out)
+        _be = (f"    end: {repr(time_begin_end)}" if time_begin_end
+               else "    end: null")
+        print(_be, file=out)
         print("  end_step:", file=out)
-        print(f"    nanosec: {end_nano}" if end_nano is not None else "    nanosec: null", file=out)
-        print(f"    begin: {repr(time_end_begin)}" if time_end_begin is not None else "    begin: null", file=out)
-        print(f"    end: {repr(time_end)}" if time_end is not None else "    end: null", file=out)
+        _en = (f"    nanosec: {end_nano}" if end_nano
+               else "    nanosec: null")
+        print(_en, file=out)
+        _eb = (f"    begin: {repr(time_end_begin)}" if time_end_begin
+               else "    begin: null")
+        print(_eb, file=out)
+        _ee = (f"    end: {repr(time_end)}" if time_end
+               else "    end: null")
+        print(_ee, file=out)
         print("data_log:", file=out)
         print(f"  total_timesteps: {total_timesteps}", file=out)
         print(f"  metrics_per_step: {metrics_per_step}", file=out)
         print(f"  log_count: {log_count}", file=out)
     finally:
         out.close()
 
-    # Only errors go to stdout: timestep and its metric count for inconsistent timesteps
+    # Only errors to stdout: timestep and metric count for inconsistent ones
     if metrics_per_step == "ERROR":
         for ts in sorted(timesteps_with_error, key=int):
             print(ts, counts_per_timestep[ts], file=sys.stdout)

roles/telemetry_chargeback/tasks/loki_rate.yml

@@ -27,15 +27,3 @@
 - name: "**INFO** Print the rating summary {{ item }}"
   ansible.builtin.debug:
     var: cost_totals_summary
-
-
-- name: "**INFO** help for openstack rating summary cmd"
-  ansible.builtin.command:
-    cmd: "{{ openstack_cmd }} rating summary get -h"
-  register: cost_totals_h
-  changed_when: true
-  failed_when: cost_totals_h.rc != 0
-
-- name: "**INFO** Print the rating summary get help output"
-  ansible.builtin.debug:
-    var: cost_totals_h.stdout_lines