[OSPRH-21020] Adjust rabbitmq tests

A bugfix proposed in openstack-k8s-operators/telemetry-operator#780
changes how rabbitmq metrics are being scraped. The FVT needs
to be adjusted for the tests to pass. For a limited time, both
versions of telemetry-operator (pre-bugfix and post-bugfix) are allowed
to pass the tests. A PR to remove the pre-bugfix tasks will be created in the
future.

Author: vyzigold

roles/telemetry_verify_metrics/tasks/main.yml

@@ -21,11 +21,31 @@
         condition_type: Ready
   tags: precheck
 
-- name: Verify RabbitMQ metrics are being exposed and stored
+# TODO: task can be deleted once well after FR4 BR1
+- name: Determine if RabbitMQ scraping uses PodMonitors
+  ansible.builtin.shell: |
+    oc get podmonitors.monitoring.rhobs telemetry-rabbitmq -o custom-columns=NAME:.metadata.name --no-headers
+  register: rabbitmq_podmonitors
+  changed_when: false
+  failed_when: false
+
+- name: Verify RabbitMQ metrics are being exposed and stored with PodMonitor
+  ansible.builtin.include_tasks:
+    file: verify_rabbitmq_metrics_podmonitor.yml
+  tags: test
+  when:
+    - '"rabbitmq" in telemetry_verify_metrics_metric_sources_to_test'
+    # TODO: condition can be deleted once well after FR4 BR1
+    - '"telemetry-rabbitmq" in rabbitmq_podmonitors.stdout_lines'
+
+# TODO: task can be deleted once well after FR4 BR1
+- name: Verify RabbitMQ metrics are being exposed and stored with ScrapeConfig
   ansible.builtin.include_tasks:
-    file: verify_rabbitmq_metrics.yml
+    file: verify_rabbitmq_metrics_scrapeconfig.yml
   tags: test
-  when: '"rabbitmq" in telemetry_verify_metrics_metric_sources_to_test'
+  when:
+    - '"rabbitmq" in telemetry_verify_metrics_metric_sources_to_test'
+    - '"telemetry-rabbitmq" not in rabbitmq_podmonitors.stdout_lines'
 
 - name: Verify Ceilometer compute metrics are being exposed and stored
   ansible.builtin.include_tasks:

roles/telemetry_verify_metrics/tasks/verify_rabbitmq_metrics_podmonitor.yml

@@ -0,0 +1,50 @@
+- name: Get rabbitmq CR names
+  ansible.builtin.shell: |
+    oc get rabbitmqs -o custom-columns=NAME:.metadata.name --no-headers
+  register: cr_names
+  changed_when: false
+  failed_when: cr_names.rc >= 1 or cr_names.stdout == ""
+
+- name: Create rabbitmq cr list
+  ansible.builtin.set_fact:
+    rabbitmq_cr_list: "{{ rabbitmq_cr_list | default([]) + [{'kind': 'podmonitors.monitoring.rhobs', 'name': 'telemetry-' + item}] }}"
+  loop: "{{ cr_names.stdout_lines }}"
+
+- name: Get rabbitmq pod names
+  ansible.builtin.shell: |
+    oc get pods -l app.kubernetes.io/component=rabbitmq -o custom-columns=NAME:.metadata.name --no-headers
+  register: pod_names
+  changed_when: false
+
+- name: Create rabbitmq pod list
+  ansible.builtin.set_fact:
+    rabbitmq_pod_list: "{{ rabbitmq_pod_list | default([]) + [item] }}"
+  loop: "{{ pod_names.stdout_lines }}"
+
+- name: Verify rabbitmq podmonitors exist and pods are running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_list: "{{ rabbitmq_cr_list }}"
+    common_pod_list: "{{ rabbitmq_pod_list }}"
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+
+- name: Check rabbitmq metric endpoints
+  ansible.builtin.shell: |
+    oc rsh openstackclient curl https://{{ item }}.openstack.svc:15691/metrics
+  register: result
+  changed_when: false
+  failed_when: result.rc >= 1
+  loop: "{{ cr_names.stdout_lines }}"
+
+- name: |
+    TEST Use openstack observabilityclient to verify rabbitmq metrics are stored in prometheus
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric query rabbitmq_identity_info{pod=\'{{ item }}\'} --disable-rbac
+  register: result
+  delay: 30
+  retries: 10
+  changed_when: false
+  until: result.rc == 0 and "rabbitmq_identity_info" in result.stdout
+  loop: "{{ rabbitmq_pod_list }}"

roles/telemetry_verify_metrics/tasks/verify_rabbitmq_metrics.yml renamed to roles/telemetry_verify_metrics/tasks/verify_rabbitmq_metrics_scrapeconfig.yml

@@ -1,3 +1,7 @@
+# TODO: Delete this file once we're sure we're running
+#       at least FR4 bugfix release 1 everywhere.
+#       As part of a bugfix, telemetry-operator uses PodMonitors for rabbit
+#       instead of ScrapeConfigs now.
 - name: Verify rabbitmq scrapeconfig exists
   ansible.builtin.include_role:
     name: common