Merge pull request #798 from stackhpc/upstream/2025.1-2025-09-29

Synchronise 2025.1 with upstream

Author: priteau

ansible/roles/ovn-db/tasks/bootstrap-db.yml

@@ -11,28 +11,48 @@
 
 - name: Get OVN_Northbound cluster leader
   become: true
-  command: "{{ kolla_container_engine }} exec ovn_nb_db ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound"
+  command: >-
+    {{ kolla_container_engine }} exec ovn_nb_db
+    ovs-appctl -t /var/run/ovn/ovnnb_db.ctl
+    cluster/status OVN_Northbound
   changed_when: False
   register: ovn_nb_cluster_status
 
 - name: Configure OVN NB connection settings
   vars:
     search_string: "Role: leader"
   become: true
-  command: "{{ kolla_container_engine }} exec ovn_nb_db ovn-nbctl --inactivity-probe={{ ovn_nb_db_inactivity_probe }} set-connection ptcp:{{ ovn_nb_db_port }}:0.0.0.0"
+  command: >-
+    {{ kolla_container_engine }} exec ovn_nb_db
+    ovn-nbctl --inactivity-probe={{ ovn_nb_db_inactivity_probe }}
+    set-connection ptcp:{{ ovn_nb_db_port }}:0.0.0.0
+  register: ovn_nb_set_connection_result
+  retries: 3
+  delay: 5
+  until: ovn_nb_set_connection_result.rc == 0
   when: ovn_nb_cluster_status is search(search_string)
 
 - name: Get OVN_Southbound cluster leader
   become: true
-  command: "{{ kolla_container_engine }} exec ovn_sb_db ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound"
+  command: >-
+    {{ kolla_container_engine }} exec ovn_sb_db
+    ovs-appctl -t /var/run/ovn/ovnsb_db.ctl
+    cluster/status OVN_Southbound
   changed_when: False
   register: ovn_sb_cluster_status
 
 - name: Configure OVN SB connection settings
   vars:
     search_string: "Role: leader"
   become: true
-  command: "{{ kolla_container_engine }} exec ovn_sb_db ovn-sbctl --inactivity-probe={{ ovn_sb_db_inactivity_probe }} set-connection ptcp:{{ ovn_sb_db_port }}:0.0.0.0"
+  command: >-
+    {{ kolla_container_engine }} exec ovn_sb_db
+    ovn-sbctl --inactivity-probe={{ ovn_sb_db_inactivity_probe }}
+    set-connection ptcp:{{ ovn_sb_db_port }}:0.0.0.0
+  register: ovn_sb_set_connection_result
+  retries: 3
+  delay: 5
+  until: ovn_sb_set_connection_result.rc == 0
   when: ovn_sb_cluster_status is search(search_string)
 
 - name: Wait for ovn-nb-db

tests/check-logs.sh

@@ -182,6 +182,22 @@ if sudo test -d /var/log/kolla; then
         fi
     done
 
+    # NOTE: Check if OpenSearch output plugin has connected in OpenSearch scenarios, otherwise
+    #       check_fluentd_missing_logs will fail because fluentd will only parse files when
+    #       output plugin is working.
+    retries=0
+    retries_max=10
+    until [[ $(sudo tail -n 5 /var/log/kolla/fluentd/fluentd.log | grep "Could not communicate to OpenSearch" | wc -l) -eq 0 ]]; do
+        echo "Found 'Could not communicate to OpenSearch' in last 5 lines of fluentd.log, sleeping 30 seconds"
+        retries=$((retries + 1))
+        if [[ $retries != $retries_max ]]; then
+            sleep 30
+        else
+            echo "Found 'Could not communicate to OpenSearch' in last 5 lines of fluentd.log after 10 retries." | tee -a $fluentd_error_summary_file
+            break
+        fi
+    done
+
     if check_fluentd_missing_logs >/dev/null; then
         any_critical=1
         echo "(critical) Found some missing log files in fluentd logs. Matches in $fluentd_error_summary_file"