qa/standalone/mon/mon_cluster_log.sh: retry check for log line

Issue: The test was failing as we were checking for the osd boot
log before it was actually emitted in the log file.

Solution: We retry checking for the desired string in the log file
for a duration of 60s after OSD has come up successfully.

Fixes: https://tracker.ceph.com/issues/67282

Signed-off-by: Shraddha Agrawal <[email protected]>
Signed-off-by: Naveen Naidu <[email protected]>

Author: shraddhaag

qa/standalone/mon/mon-cluster-log.sh

@@ -62,7 +62,7 @@ function TEST_cluster_log_level() {
     ceph config set mon.a mon_cluster_log_level info
     ceph osd down 0
     TIMEOUT=20 wait_for_osd up 0 || return 1
-    grep -q "cluster [[]INF[]] osd.0.*boot" $dir/log
+    TIMEOUT=60 wait_for_string $dir/log "cluster [[]INF[]] osd.0.*boot" 
     return_code=$?
     if [ $return_code -ne 0 ]; then
       echo "Failed : Could not find INF log in the cluster log file"
@@ -145,9 +145,17 @@ function TEST_journald_cluster_log_level() {
     ceph osd down 0
     TIMEOUT=20 wait_for_osd up 0 || return 1
     search_str="osd.0.*boot"
-    sudo journalctl _COMM=ceph-mon CEPH_CHANNEL=cluster PRIORITY=6 --output=json-pretty --since "60 seconds ago" |jq '.MESSAGE' > $dir/journal.log
-    grep -q "$search_str" $dir/journal.log
-    return_code=$?
+    return_code=1
+    RETRY_DURATION=60 
+    for ((i=0; i < $RETRY_DURATION; i++)); do
+      sudo journalctl _COMM=ceph-mon CEPH_CHANNEL=cluster PRIORITY=6 --output=json-pretty --since "60 seconds ago" |jq '.MESSAGE' > $dir/journal.log
+      if ! grep "$search_str" $dir/journal.log; then
+        sleep 1
+      else
+        return_code=0
+        break
+      fi
+    done
     if [ $return_code -ne 0 ]; then
       echo "Failed : Could not find INF log in the journalctl log file"
       ERRORS=$(($ERRORS + 1))