Add task files to push and retrieve ck data log to loki

roles/telemetry_chargeback/README.md

@@ -2,10 +2,13 @@ telemetry_chargeback
 =========
 The **`telemetry_chargeback`** role is designed to test the **RHOSO Cloudkitty** feature. These tests are specific to the Cloudkitty feature. Tests that are not specific to this feature (e.g., standard OpenStack deployment validation, basic networking) should be added to a common role.
 
-The role performs two main functions:
+The role performs four main functions:
 
 1. **CloudKitty Validation** - Enables and configures the CloudKitty hashmap rating module, then validates its state.
-2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and Jinja2 template.
+2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and 
+Jinja2 template.
+3. **Ingest data and Flush to Loki** - Ingests synthetic CloudKitty log data and Flush Loki Ingester Memory to Storage
+4. **Retrieval of data** - Verifies retrieval of data from loki
 
 Requirements
 ------------
@@ -47,6 +50,7 @@ These variables are used internally by the role and typically do not need to be
 | `ck_data_config` | `{{ role_path }}/files/test_static.yml` | Path to the scenario configuration file. |
 | `ck_output_file_local` | `{{ artifacts_dir_zuul }}/loki_synth_data.json` | Local path for generated synthetic data. |
 | `ck_output_file_remote` | `{{ logs_dir_zuul }}/gen_loki_synth_data.log` | Remote destination for synthetic data. |
+| `ck_loki_retrieve_file` | `{{ logs_dir_zuul }}/retrieve_loki_op.json` | Path where the retrieval of loki data is stored. |
 
 Scenario Configuration
 ----------------------
@@ -62,6 +66,25 @@ Dependencies
 ------------
 This role has no direct hard dependencies on other Ansible roles.
 
+This runs 5 playbooks
+---------------------
+```yaml
+- name: "Validate Chargeback Feature"
+  ansible.builtin.include_tasks: "chargeback_tests.yml"
+
+- name: "Generate Synthetic Data"
+  ansible.builtin.include_tasks: "gen_synth_loki_data.yml"
+
+- name: "Ingests Cloudkitty Data log"
+  ansible.builtin.include_tasks: "ingest_loki_data.yml"
+
+- name: "Flush Data to loki Storage"
+  ansible.builtin.include_tasks: "flush_loki_data.yml"
+
+- name: "Retrieve Data log from loki"
+  ansible.builtin.include_tasks: "retrieve_loki_data.yml"
+```
+
 Example Playbook
 ----------------
 ```yaml
@@ -78,4 +101,4 @@ Example Playbook
 Author Information
 ------------------
 
-Alex Yefimov, Red Hat
+Alex Yefimov, Muneesha Yadla

roles/telemetry_chargeback/tasks/flush_loki_data.yml

@@ -0,0 +1,79 @@
+---
+# Flush Loki Ingester Memory to Storage
+
+# create dir 
+- name: Create a directory to extract certificates
+  ansible.builtin.file:
+    path: "{{ local_cert_dir }}"
+    state: directory
+    mode: '0755'
+
+# Extract Certs
+- name: Extract Client Certificates
+  ansible.builtin.command:
+    cmd: "oc extract {{ client_secret }} --to={{ local_cert_dir }} --confirm -n {{ namespace }}"
+  changed_when: true
+
+# Extract CA Bundle
+- name: Extract CA Bundle
+  ansible.builtin.command:
+    cmd: "oc extract {{ ca_configmap }} --to={{ local_cert_dir }} --confirm -n {{ namespace }}"
+  changed_when: true
+
+# Flush
+- name: Flush Execution inside openstack CLI
+  block:
+    # create dir 
+    - name: Create directory inside openstack CLI
+      ansible.builtin.command:
+        cmd: "oc exec -n {{ namespace }} {{ openstackpod }} -- mkdir -p {{ remote_cert_dir }}"
+      changed_when: false
+
+    # copy all certs
+    - name: Copy certificates to openstack CLI
+      ansible.builtin.command:
+        cmd: "oc cp {{ local_cert_dir }}/. {{ namespace }}/{{ openstackpod }}:{{ remote_cert_dir }}/"
+      changed_when: true
+
+    # flush loki
+    - name: Trigger Flush
+      ansible.builtin.command:
+        cmd: >
+          oc exec -n {{ namespace }} {{ openstackpod }} --
+          curl -v -X POST {{ ingester_flush_url }}
+          --cert {{ remote_cert_dir }}/tls.crt
+          --key {{ remote_cert_dir }}/tls.key
+          --cacert {{ remote_cert_dir }}/service-ca.crt
+      register: flush_response
+      changed_when: true
+      failed_when: flush_response.rc != 0
+
+    # Status
+    - name: Verify Flush Status
+      ansible.builtin.assert:
+        that:
+          - "'204' in flush_response.stderr or '200' in flush_response.stderr"
+        fail_msg: "Flush failed"
+        success_msg: "Ingester Memory Flushed successfully"
+
+    - name: Confirm Success
+      ansible.builtin.debug:
+        msg: "Loki Ingester flushed the data successfully"
+
+  rescue:
+    - name: Debug Failure Output
+      ansible.builtin.debug:
+        msg: 
+          - "Failure"
+          - "Stdout: {{ flush_response.stdout }}"
+          - "Stderr: {{ flush_response.stderr }}"
+
+    - name: Report Failure
+      ansible.builtin.fail:
+        msg: "Flush failure"
+
+# Cleanup
+- name: Cleanup local certificates
+  ansible.builtin.file:
+    path: "{{ local_cert_dir }}"
+    state: absent

roles/telemetry_chargeback/tasks/ingest_loki_data.yml

@@ -0,0 +1,59 @@
+---
+# Ingest data log to Loki that is generated from gen_synth_loki_data.yml"
+
+# Create a directory to add certs
+- name: Create certificates directory
+  ansible.builtin.file:
+    path: "{{ cert_dir }}"
+    state: directory
+    mode: '0755'
+
+# Extract certificates
+- name: Extract certificates from openshift secret
+  ansible.builtin.command:
+    cmd: >
+      oc extract secret/{{ cert_secret_name }}
+      --to={{ cert_dir }}
+      --confirm
+  changed_when: true
+
+# Push the json format data log to loki
+- name: Ingest data log to Loki via API
+  block:
+
+    - name: Read log file content
+      ansible.builtin.slurp:
+        src: "{{ ck_output_file_remote }}"
+      register: log_file_content
+
+    - name: Push data to Loki
+      ansible.builtin.uri:
+        url: "{{ loki_push_url }}"
+        method: POST
+        body: "{{ log_file_content['content'] | b64decode | from_json }}"
+        body_format: json
+        client_cert: "{{ cert_dir }}/tls.crt"
+        client_key: "{{ cert_dir }}/tls.key"
+        validate_certs: false
+        status_code: 204
+        return_content: yes
+      register: loki_response
+      ignore_errors: false
+      failed_when: loki_response.status != 204
+
+    # Success
+    - name: Confirm Success
+      ansible.builtin.debug:
+        msg: "Ingestion Successful!"
+
+  rescue:
+    # Rescue block
+    - name: Debug failure
+      ansible.builtin.debug:
+        msg: "{{ loki_response.status }}"
+
+    # Failure
+    - name: Report Ingestion Failure
+      ansible.builtin.fail:
+        msg: "Ingestion Failed"
+      ignore_errors: false

roles/telemetry_chargeback/tasks/main.yml

@@ -4,3 +4,12 @@
 
 - name: "Generate Synthetic Data"
   ansible.builtin.include_tasks: "gen_synth_loki_data.yml"
+
+- name: "Ingests Cloudkitty Data log"
+  ansible.builtin.include_tasks: "ingest_loki_data.yml"
+
+- name: "Flush Data to loki Storage"
+  ansible.builtin.include_tasks: "flush_loki_data.yml"
+
+- name: "Retrieve Data log from loki"
+  ansible.builtin.include_tasks: "retrieve_loki_data.yml"

roles/telemetry_chargeback/tasks/retrieve_loki_data.yml

@@ -0,0 +1,92 @@
+---
+# Automate Loki Log Retrieval
+
+# Count the entries in the log file
+- name: Read the json data log file to calculate the no. of entries
+  ansible.builtin.slurp:
+    src: "{{ ck_output_file_remote }}"
+  register: source_file_data
+
+- name: Set Expected Log Count
+  ansible.builtin.set_fact:
+    expected_log_count: "{{ (source_file_data['content'] | b64decode | from_json).streams | map(attribute='values') | map('length') | sum }}"
+
+- name: Expected Count
+  ansible.builtin.debug:
+    msg: "Input file has {{ expected_log_count }} data entries that Loki has to return"
+
+# Calculate Time
+- name: Calculate Start Time in nanoseconds
+  ansible.builtin.command: date -d "{{ lookback_days }} days ago" +%s000000000
+  register: nano_time
+  changed_when: false
+
+- name: Set Start Time
+  ansible.builtin.set_fact:
+    start_time: "{{ nano_time.stdout }}"
+
+- name: Display Query Parameters
+  ansible.builtin.debug:
+    msg: 
+      - "Query: {{ logql_query }}"
+      - "Start Time: {{ start_time }}"
+      - "Limit: {{ limit_logs }}"
+
+# Query Loki
+- name: Retrieve Logs from Loki via API
+  block:
+    - name: Query Loki API
+      ansible.builtin.uri:
+        url: "{{ loki_query_url }}?query={{ logql_query | urlencode }}&start={{ start_time }}&limit={{ limit_logs }}"
+        method: GET
+        client_cert: "{{ cert_dir }}/tls.crt"
+        client_key: "{{ cert_dir }}/tls.key"
+        ca_path: "{{ cert_dir }}/ca.crt"
+        validate_certs: false
+        return_content: yes
+        body_format: json
+      register: loki_response
+      # Wait condition
+      until:
+        - loki_response.status == 200
+        - loki_response.json.status == 'success'
+        - loki_response.json.data.result | length > 0
+        - (loki_response.json.data.result | map(attribute='values') | map('length') | sum) >= expected_log_count|int
+      retries: 5
+      delay: 300
+
+    # Save data
+    - name: Save Loki Data to JSON file
+      ansible.builtin.copy:
+        content: "{{ loki_response.json | to_nice_json }}"
+        dest: "{{ ck_loki_retrieve_file }}"
+        mode: '0644'
+
+    # Validate
+    - name: Verify Data Integrity
+      ansible.builtin.assert:
+        that:
+          - loki_response.json.status == 'success'
+          - loki_response.json.data.result | length > 0
+        fail_msg: "Query did not return any data entries"
+        success_msg: "Query found all the data entries"
+
+    # Success
+    - name: Confirm Success
+      vars:
+        actual_count: "{{ loki_response.json.data.result | map(attribute='values') | map('length') | sum }}"
+      ansible.builtin.debug:
+        msg: "Successful, Input file had {{ expected_log_count }} entries and Loki returned {{ actual_count }}"
+
+  rescue:
+    - name: Debug failure
+      ansible.builtin.debug:
+        msg: 
+          - "Status: {{ loki_response.status | default('Unknown') }}"
+          - "Body: {{ loki_response.content | default('No Content') }}"
+          - "Msg: {{ loki_response.msg | default('Request failed') }}"
+
+    # Failure
+    - name: Report Retrieval Failure
+      ansible.builtin.fail:
+        msg: "Retrieval Failed"

roles/telemetry_chargeback/vars/main.yml

@@ -7,3 +7,32 @@ ck_data_template: "{{ role_path }}/template/loki_data_templ.j2"
 ck_data_config: "{{ role_path }}/files/test_static.yml"
 ck_output_file_local: "{{ artifacts_dir_zuul }}/loki_synth_data.json"
 ck_output_file_remote: "{{ logs_dir_zuul }}/gen_loki_synth_data.log"
+ck_loki_retrieve_file: "{{ logs_dir_zuul }}/retrieve_loki_op.json"
+
+# loki url's
+loki_base_url: "https://cloudkitty-lokistack-openstack.apps-crc.testing"
+loki_push_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/push"
+loki_query_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/query_range"
+
+# Ingester URL
+ingester_flush_url: "https://cloudkitty-lokistack-ingester-http.openstack.svc:3100/flush"
+
+# Cloudkitty certificates
+cert_secret_name: "cert-cloudkitty-client-internal"
+cert_dir: "{{ ansible_user_dir }}/ck-certs"
+
+client_secret: "secret/cloudkitty-lokistack-gateway-client-http"
+ca_configmap: "cm/cloudkitty-lokistack-ca-bundle"
+local_cert_dir: "{{ ansible_env.HOME }}/flush_certs"
+remote_cert_dir: "osp-certs"
+
+# LogQL Query
+logql_query: "{{ loki_query | default('{service=\"cloudkitty\"}') }}"
+
+# Time window settings
+lookback_days: "{{ lookback | default(6) }}"
+limit_logs: "{{ limit | default(50) }}"
+
+# vars
+namespace: "openstack"
+openstackpod: "openstackclient"