Merge pull request ceph#60511 from rhcs-dashboard/health-test-fix

qa/dashboard: fix tasks.mgr.dashboard.test_health.HealthTest

Reviewed-by: Ernesto Puerta <[email protected]>

Author: nizamial09

qa/tasks/mgr/dashboard/helper.py

@@ -220,13 +220,11 @@ def setUpClass(cls):
 
             # To avoid any issues with e.g. unlink bugs, we destroy and recreate
             # the filesystem rather than just doing a rm -rf of files
-            cls.mds_cluster.mds_stop()
-            cls.mds_cluster.mds_fail()
             cls.mds_cluster.delete_all_filesystems()
+            cls.mds_cluster.mds_restart()  # to reset any run-time configs, etc.
             cls.fs = None  # is now invalid!
 
             cls.fs = cls.mds_cluster.newfs(create=True)
-            cls.fs.mds_restart()
 
             # In case some test messed with auth caps, reset them
             # pylint: disable=not-an-iterable

qa/tasks/mgr/dashboard/test_mgr_module.py

@@ -4,6 +4,7 @@
 import logging
 
 import requests
+from urllib3.exceptions import MaxRetryError
 
 from .helper import (DashboardTestCase, JLeaf, JList, JObj,
                      module_options_object_schema, module_options_schema,
@@ -24,10 +25,11 @@ def wait_until_rest_api_accessible(self):
         def _check_connection():
             try:
                 # Try reaching an API endpoint successfully.
+                logger.info('Trying to reach the REST API endpoint')
                 self._get('/api/mgr/module')
                 if self._resp.status_code == 200:
                     return True
-            except requests.ConnectionError:
+            except (MaxRetryError, requests.ConnectionError):
                 pass
             return False
 

qa/tasks/mgr/mgr_test_case.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import socket
 
 from unittest import SkipTest
 
@@ -229,15 +230,22 @@ def _assign_ports(cls, module_name, config_name, min_port=7789):
         """
         # Start handing out ports well above Ceph's range.
         assign_port = min_port
+        ip_addr = cls.mgr_cluster.get_mgr_map()['active_addr'].split(':')[0]
 
         for mgr_id in cls.mgr_cluster.mgr_ids:
             cls.mgr_cluster.mgr_stop(mgr_id)
             cls.mgr_cluster.mgr_fail(mgr_id)
 
+
         for mgr_id in cls.mgr_cluster.mgr_ids:
-            log.debug("Using port {0} for {1} on mgr.{2}".format(
-                assign_port, module_name, mgr_id
-            ))
+            # Find a port that isn't in use
+            while True:
+                if not cls.is_port_in_use(ip_addr, assign_port):
+                    break
+                log.debug(f"Port {assign_port} in use, trying next")
+                assign_port += 1
+
+            log.debug(f"Using port {assign_port} for {module_name} on mgr.{mgr_id}")
             cls.mgr_cluster.set_module_localized_conf(module_name, mgr_id,
                                                       config_name,
                                                       str(assign_port),
@@ -255,3 +263,8 @@ def is_available():
                     mgr_map['active_name'], mgr_map['active_gid']))
             return done
         cls.wait_until_true(is_available, timeout=30)
+
+    @classmethod
+    def is_port_in_use(cls, ip_addr: str, port: int) -> bool:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex((ip_addr, port)) == 0

src/pybind/mgr/dashboard/run-backend-api-tests.sh

@@ -134,7 +134,7 @@ run_teuthology_tests() {
     export CEPH_OUT_CLIENT_DIR=${LOCAL_BUILD_DIR}/out/client
     find . -iname "*${COVERAGE_FILE}*" -type f -delete
 
-    python ../qa/tasks/vstart_runner.py --ignore-missing-binaries --no-verbose $OPTIONS $(echo $TEST_CASES) ||
+    python ../qa/tasks/vstart_runner.py --ignore-missing-binaries --no-verbose --debug $OPTIONS $(echo $TEST_CASES) ||
       on_tests_error
 
     deactivate