[test_alerts] Replace pause with retry/until (#198)

* Replace pause with a continuous check
* Extend the wait time, to account for latency

Author: elfiesmelfie

roles/test_alerts/tasks/test_creating_a_standard_alert_route_in_alert_manager.yml

@@ -3,6 +3,13 @@
 
 # Pre-check: is the value of global.timeout = 5m in the alertmanager secret
 
+- name: "Get the number of default-interconnect pods"
+  ansible.builtin.command:
+    cmd: |
+      oc get pods -l application=default-interconnect
+  register: expected_pods
+  changed_when: false
+
 - name: "Do the test procedure"
   block:
       # TODO: put the patch into a file. and use --patch-file instead of -p OR slurp the file from files/
@@ -101,10 +108,16 @@
       until: 'not "FVT_TESTING Collectd metrics receive rate is zero" in cmd_output.stdout'
       changed_when: false
 
-    - name: "Wait 2 minutes to make sure all SG pods are back to normal"
-      ansible.builtin.pause:
-        minutes: 2
-
+    - name: "Wait up to 5 minutes to make sure all default-interconnect pods are back"
+      ansible.builtin.command:
+        cmd: |
+          oc get pods -l application=default-interconnect
+      retries: 30
+      delay: 10
+      register: output
+      until: output.stdout_lines | length == expected_pods.stdout_lines | length
+      changed_when: false
+      
 
     - name: "RHELOSP-176039 Remove alertmanagerConfigManifest from the ServiceTelemetry object"
       ansible.builtin.shell: