qa/upgrade: fix checks to make sure upgrade is still in progress

Without checking both for the upgrade being in progress and that
the status isn't reporting an error, we can end up in a scenario
where the test is just waiting for an upgrade that has already
been marked failed and will never complete. This same sort of
change was already done in the orch suite upgrade tests and
has helped with jobs timing out there

Fixes: https://tracker.ceph.com/issues/65546

Signed-off-by: Adam King <[email protected]>

Author: adk3798

qa/suites/upgrade/quincy-x/parallel/upgrade-sequence.yaml

@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

qa/suites/upgrade/quincy-x/stress-split/1-start.yaml

@@ -58,28 +58,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 

qa/suites/upgrade/reef-x/parallel/upgrade-sequence.yaml

@@ -7,7 +7,7 @@ upgrade-sequence:
        mon.a:
          - ceph config set global log_to_journald false --force
          - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+         - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
          - ceph orch ps
          - ceph versions
          - ceph versions | jq -e '.overall | length == 1'

qa/suites/upgrade/reef-x/stress-split/1-start.yaml

@@ -57,28 +57,28 @@ first-half-sequence:
       - ceph orch ps
 
       - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
+      - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
+      - "while ! ceph mon versions | grep $sha1 | grep ': 3' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do sleep 2 ; done"
       - ceph orch ps
       - ceph orch upgrade pause
       - sleep 60
       - ceph orch upgrade resume
 
       - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
+      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]' && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done"
       - ceph orch upgrade pause
       - ceph orch ps
 
@@ -112,7 +112,7 @@ second-half-sequence:
           - sleep 60
 
           - echo wait for upgrade to complete
-          - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+          - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
 
           - echo upgrade complete
           - ceph orch ps

qa/suites/upgrade/telemetry-upgrade/quincy-x/1-tasks.yaml

@@ -56,7 +56,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'

qa/suites/upgrade/telemetry-upgrade/reef-x/1-tasks.yaml

@@ -55,7 +55,7 @@ tasks:
         mon.a:
             - ceph config set global log_to_journald false --force
             - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-            - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
+            - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
             - ceph orch ps
             - ceph versions
             - ceph versions | jq -e '.overall | length == 1'