Add metrics verification tasks for power monitoring components (#193)

Author: yadneshk

.zuul.yaml

@@ -70,6 +70,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: *required_projects

roles/common/README.md

@@ -155,6 +155,7 @@ can be set at the play level.
                 - ceilometer_agent_compute
                 - ceilometer_agent_ipmi
                 - node_exporter
+                - kepler
 
       - name: "Verify Endpoint"
         ansible.builtin.import_role:

roles/telemetry_verify_metrics/README.md

@@ -24,13 +24,16 @@ Tests:
 - Verify Ceilometer metrics are being exposed and stored
     - Use openstack observabilityclient to verify Ceilometer central metrics are stored in Prometheus
     - Use openstack observabilityclient to verify Ceilometer compute metrics are stored in Prometheus
+    - Use openstack observabilityclient to verify Ceilometer ipmi metrics are stored in Prometheus
 - Verify NodeExporter metrics are being exposed and stored
     - Use openstack observabilityclient to verify NodeExporter metrics are stored in Prometheus
+- Verify Kepler metrics are being exposed and stored
+    - Use openstack observabilityclient to verify Kepler metrics are stored in Prometheus
 
 Role Variables
 --------------
 openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
-telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, node\_exporter, rabbitmq
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler
 
 Example Playbook
 ----------------

roles/telemetry_verify_metrics/defaults/main.yml

@@ -4,3 +4,6 @@ telemetry_verify_metrics_metric_sources_to_test:
   - ceilometer_central_agent
   - node_exporter
   - rabbitmq
+  - kepler
+  # NOTE: IPMI metrics can only be verified when compute nodes are baremetal
+  # - ceilometer_ipmi_agent

roles/telemetry_verify_metrics/tasks/main.yml

@@ -41,8 +41,22 @@
   tags: test
   when: '"ceilometer_central_agent" in telemetry_verify_metrics_metric_sources_to_test'
 
+- name: Verify Ceilometer ipmi metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_ipmi_metrics.yml
+  tags: power_monitoring
+  when: '"ceilometer_ipmi_agent" in telemetry_verify_metrics_metric_sources_to_test'
+
 - name: Verify NodeExporter metrics are being exposed and stored
   ansible.builtin.include_tasks:
     file: verify_node_exporter_metrics.yml
   tags: test
   when: '"node_exporter" in telemetry_verify_metrics_metric_sources_to_test'
+
+- name: Verify Kepler metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_kepler_metrics.yml
+  tags: 
+    - test
+    - power_monitoring
+  when: '"kepler" in telemetry_verify_metrics_metric_sources_to_test'

roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml

@@ -0,0 +1,40 @@
+- name: Verify ceilometer scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1221
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ceilometer
+
+- name: Verify ceilometer ipmi agent container is up
+  ansible.builtin.include_tasks:
+    file: check_compute_node_containers.yml
+  vars:
+    common_container_test_id: RHOSO-1230
+    common_container_list:
+      - ceilometer_agent_ipmi
+  loop: "{{ groups['computes'] }}"
+  loop_control:
+    loop_var: compute_node
+
+- name: Verify ceilometer controlplane pod is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_test_id: RHOSO-1242
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - ceilometer-0
+
+- name: |
+    TEST Use openstack observabilityclient to verify ceilometer ipmi metrics are stored in prometheus
+    RHOSO-1217
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show ceilometer_hardware_ipmi_fan --disable-rbac
+  register: result
+  delay: 30
+  retries: 10
+  until: result.rc == 0 and "ceilometer_hardware_ipmi" in result.stdout
+  changed_when: false

roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml

@@ -0,0 +1,36 @@
+- name: Verify node exporter scrapeconfigs exist
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1222
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-kepler
+    kepler_dashboard_metrics:
+      - kepler_vm_platform_joules_total
+      - kepler_node_platform_joules_total
+      - kepler_process_joules_total
+      - kepler_container_joules_total
+      - kepler_node_info
+
+- name: Verify Kepler container is up
+  ansible.builtin.include_tasks:
+    file: check_compute_node_containers.yml
+  vars:
+    common_container_test_id: RHOSO-1231
+    common_container_list:
+      - kepler
+  loop: "{{ groups['computes'] }}"
+  loop_control:
+    loop_var: compute_node
+
+- name: |
+    TEST Use openstack observabilityclient to verify kepler metrics are stored in prometheus
+    RHOSO-1216
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show --disable-rbac kepler_node_info
+  register: result
+  delay: 30
+  retries: 10
+  changed_when: false
+  until: result.rc == 0 and "kepler_node_info" in result.stdout