qa/mgr/dashboard: fix test race condition

Fixes: https://tracker.ceph.com/issues/66844

Signed-off-by: Ernesto Puerta <[email protected]>

Author: epuertat

qa/tasks/mgr/dashboard/helper.py

@@ -9,7 +9,8 @@
 import string
 import time
 from collections import namedtuple
-from typing import List
+from functools import wraps
+from typing import List, Optional, Tuple, Type, Union
 
 import requests
 from tasks.mgr.mgr_test_case import MgrTestCase
@@ -343,16 +344,16 @@ def _get(cls, url, params=None, version=DEFAULT_API_VERSION, set_cookies=False,
 
     @classmethod
     def _view_cache_get(cls, url, retries=5):
-        retry = True
-        while retry and retries > 0:
-            retry = False
+        _retry = True
+        while _retry and retries > 0:
+            _retry = False
             res = cls._get(url, version=DEFAULT_API_VERSION)
             if isinstance(res, dict):
                 res = [res]
             for view in res:
                 assert 'value' in view
                 if not view['value']:
-                    retry = True
+                    _retry = True
             retries -= 1
         if retries == 0:
             raise Exception("{} view cache exceeded number of retries={}"
@@ -722,3 +723,25 @@ def _validate_json(val, schema, path=[]):
         return _validate_json(val, JLeaf(schema), path)
 
     assert False, str(path)
+
+
+def retry(
+        on_exception: Union[Type[Exception], Tuple[Type[Exception], ...]],
+        tries=3,
+        delay=0,
+        logger: Optional[logging.Logger] = None,
+):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for i in range(tries):
+                try:
+                    return func(*args, **kwargs)
+                except on_exception as e:
+                    err = e
+                    if logger:
+                        logger.warn(f"Retried #{i+1}/{tries}: '{func.__name__}' raised '{e}'")
+                    time.sleep(delay)
+            raise err
+        return wrapper
+    return decorator

qa/tasks/mgr/dashboard/test_osd.py

@@ -5,7 +5,7 @@
 import json
 
 from .helper import (DashboardTestCase, JAny, JLeaf, JList, JObj, JTuple,
-                     devices_schema)
+                     devices_schema, log, retry)
 
 
 class OsdTest(DashboardTestCase):
@@ -283,13 +283,18 @@ def test_get_indiv_flag(self):
                     if osd['osd'] == osd_initial['osd']:
                         self.assertGreater(len(osd['flags']), len(osd_initial['flags']))
 
-        self._ceph_cmd(['osd', 'unset-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
-        flags_removed = self._get('/api/osd/flags/individual')
-        self.assertStatus(200)
-        for osd in flags_removed:
-            if osd['osd'] in [0, 1, 2]:
-                self.assertNotIn('noout', osd['flags'])
-                self.assertNotIn('noin', osd['flags'])
+        ret = self._ceph_cmd_result(['osd', 'unset-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
+        self.assertEqual(ret, 0)
+
+        @retry(on_exception=AssertionError, tries=2, delay=0.5, logger=log)
+        def check_osd_flags():
+            flags_removed = self._get('/api/osd/flags/individual')
+            self.assertStatus(200)
+            for osd in flags_removed:
+                if osd['osd'] in [0, 1, 2]:
+                    self.assertNotIn('noout', osd['flags'])
+                    self.assertNotIn('noin', osd['flags'])
+        check_osd_flags()
 
     def test_add_indiv_flag(self):
         flags_update = {'noup': None, 'nodown': None, 'noin': None, 'noout': True}