Add test for health monitoring feature (#234)

* Add test for health monitoring feature

* Disable RBAC for KSM metrics

Author: paramite

roles/telemetry_verify_metrics/defaults/main.yml

@@ -2,7 +2,9 @@
 telemetry_verify_metrics_metric_sources_to_test:
   - ceilometer_compute_agent
   - ceilometer_central_agent
+  - ksm
   - node_exporter
+  - podman_exporter
   - rabbitmq
   - kepler
   - openstack_network_exporter

roles/telemetry_verify_metrics/tasks/main.yml

@@ -47,12 +47,24 @@
   tags: power_monitoring
   when: '"ceilometer_ipmi_agent" in telemetry_verify_metrics_metric_sources_to_test'
 
+- name: Verify kube-state-metrics metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ksm_metrics.yml
+  tags: test
+  when: '"ksm" in telemetry_verify_metrics_metric_sources_to_test'
+
 - name: Verify NodeExporter metrics are being exposed and stored
   ansible.builtin.include_tasks:
     file: verify_node_exporter_metrics.yml
   tags: test
   when: '"node_exporter" in telemetry_verify_metrics_metric_sources_to_test'
 
+- name: Verify Podman Exporter metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_podman_exporter_metrics.yml
+  tags: test
+  when: '"podman_exporter" in telemetry_verify_metrics_metric_sources_to_test'
+
 - name: Verify Kepler metrics are being exposed and stored
   ansible.builtin.include_tasks:
     file: verify_kepler_metrics.yml

roles/telemetry_verify_metrics/tasks/verify_ksm_metrics.yml

@@ -0,0 +1,30 @@
+- name: Verify KSM scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ksm
+
+- name: Verify kube-state-metrics agent is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - kube-state-metrics-0
+
+- name: |
+    TEST Use openstack observabilityclient to verify to verify health reporting related metrics from kube-state-metrics are stored in Prometheus
+    RHOSO-5678
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show --disable-rbac {{ item }}
+  register: result
+  delay: 30
+  retries: 10
+  until: result.rc == 0 and "{{ item }}" in result.stdout
+  changed_when: false
+  loop:
+    - kube_pod_status_phase
+    - kube_pod_status_ready

roles/telemetry_verify_metrics/tasks/verify_node_exporter_metrics.yml

@@ -30,3 +30,14 @@
   retries: 10
   changed_when: false
   until: result.rc == 0 and "node_exporter_build_info" in result.stdout
+
+- name: |
+    TEST Use openstack observabilityclient to verify health reporting related metric from systemd collector is stored in prometheus
+    RHOSO-7890
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show --disable-rbac node_systemd_unit_state
+  register: result
+  delay: 30
+  retries: 10
+  changed_when: false
+  until: result.rc == 0 and "node_systemd_unit_state" in result.stdout

roles/telemetry_verify_metrics/tasks/verify_podman_exporter_metrics.yml

@@ -0,0 +1,35 @@
+- name: Verify podman exporter scrapeconfigs exist
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-podman-exporter
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-podman-exporter-tls
+
+- name: Verify podman exporter container is up
+  ansible.builtin.include_tasks:
+    file: check_compute_node_containers.yml
+  vars:
+    common_container_test_id: RHOSO-1231
+    common_container_list:
+      - podman_exporter
+  loop: "{{ groups['computes'] }}"
+  loop_control:
+    loop_var: compute_node
+
+- name: |
+    TEST Use openstack observabilityclient to verify health reporting related metrics from podman exporter are stored in Prometheus
+    RHOSO-12345
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show --disable-rbac {{ item }}
+  register: result
+  delay: 30
+  retries: 10
+  changed_when: false
+  until: result.rc == 0 and "{{ item }}" in result.stdout
+  loop:
+    - podman_container_info
+    - podman_container_state
+    - podman_container_health