feat: add retries to delete_node

Author: KryukovaPolina

integration-tests/robot/tests/shared/lib/ZookeeperLibrary.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import os
+import time
 
 from kazoo.client import KazooClient
 from kazoo.security import make_digest_acl, make_acl
+from kazoo.exceptions import ConnectionLoss, ConnectionClosedError
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
@@ -264,17 +266,37 @@ def update_node_value(self, zk, node_path, new_value):
         stat = zk.set(node_path, new_value.encode())
         logger.debug('Node "{}" is updated: {}'.format(node_path, stat))
 
-    def delete_node(self, zk, node_path):
+    def delete_node(self, zk, node_path, retries=3, delay=5, backoff=2):
         """
         Delete the node.
         *Args:*\n
             _zk_ (KazooClient) - ZooKeeper client;\n
             _node_path_ (str) - path of the node;\n
+            _retries_ (int, optional) - number of retry attempts (default: 5);\n
+            _delay_ (int, optional) - initial delay between retries in seconds (default: 2);\n
+            _backoff_ (int, optional) - backoff multiplier (default: 2);\n
+
         *Example:*\n
-            | Delete Node | zk | /zookeeper_crud/tests |
+            | Delete Node | zk | /zookeeper_crud/tests | retries=3 | delay=1 |
         """
-        zk.delete(node_path, recursive=True)
-        logger.debug('Node "{}" is deleted.'.format(node_path))
+        for attempt in range(1, retries + 1):
+            try:
+                if zk.exists(node_path):
+                    zk.delete(node_path, recursive=True)
+                    logger.debug(f'Node "{node_path}" is deleted.')
+                    return
+                else:
+                    logger.debug(f'Node "{node_path}" does not exist.')
+                    return
+
+            except ConnectionLoss as e:
+                msg = (f'Attempt {attempt}/{retries}: cannot delete node "{node_path}" '
+                       f'due to ConnectionLoss: {e}')
+                logger.warn(msg)
+                BuiltIn().log_to_console(msg)
+                if attempt == retries:
+                    self.builtin.fail(f'Failed to delete node "{node_path}" after {retries} attempts: {e}')
+                time.sleep(delay)
 
     def find_minimum(self, first, second):
         """

integration-tests/robot/tests/zookeeper/acl_tests/acl_tests.robot

@@ -6,16 +6,14 @@ ${ADMIN_PASSWORD}             admin
 ${USERNAME}                   user123
 ${PASSWORD}                   pwd123
 ${ACL_VALUE}                  ACL
-${SUITE_RETRY_TIME}           2min
-${SUITE_RETRY_INTERVAL}       10s
 
 *** Settings ***
 Library  String
 Library  Collections
 Library  RetryFailed
 Resource  ../../shared/keywords.robot
-Suite Setup  Wait Until Keyword Succeeds  ${SUITE_RETRY_TIME}  ${SUITE_RETRY_INTERVAL}  Setup
-Suite Teardown  Wait Until Keyword Succeeds  ${SUITE_RETRY_TIME}  ${SUITE_RETRY_INTERVAL}  Cleanup
+Suite Setup  Setup
+Suite Teardown  Cleanup
 Test Teardown  Run Keyword If Test Failed  Sleep  5s
 
 *** Keywords ***

integration-tests/robot/tests/zookeeper/sasl/sasl_tests.robot

@@ -6,16 +6,14 @@ ${ZOOKEEPER_ADMIN_PASSWORD}   %{ZOOKEEPER_ADMIN_PASSWORD}
 ${ZOOKEEPER_CLIENT_USERNAME}  %{ZOOKEEPER_CLIENT_USERNAME}
 ${ZOOKEEPER_CLIENT_PASSWORD}  %{ZOOKEEPER_CLIENT_PASSWORD}
 ${ACL_VALUE}                  ACL
-${SUITE_RETRY_TIME}           2min
-${SUITE_RETRY_INTERVAL}       10s
 
 *** Settings ***
 Library  String
 Library  Collections
 Library  RetryFailed
 Resource  ../../shared/keywords.robot
-Suite Setup  Wait Until Keyword Succeeds  ${SUITE_RETRY_TIME}  ${SUITE_RETRY_INTERVAL}  Setup
-Suite Teardown  Wait Until Keyword Succeeds  ${SUITE_RETRY_TIME}  ${SUITE_RETRY_INTERVAL}  Cleanup
+Suite Setup  Setup
+Suite Teardown  Cleanup
 Test Teardown  Run Keyword If Test Failed  Sleep  5s
 
 *** Keywords ***

monitoring/exec-scripts/health_metric.py

@@ -112,15 +112,15 @@ def get_server_state(server_address):
                 logger.info(f"Server {zookeeper_server} state: {state}")
                 return state
             else:
-                logger.warning(f"Unexpected output format from {zookeeper_server}: {stdout}")
+                logger.warn(f"Unexpected output format from {zookeeper_server}: {stdout}")
         else:
-            logger.warning(f"Failed to retrieve state information from {zookeeper_server} (return code: {process.returncode})")
+            logger.warn(f"Failed to retrieve state information from {zookeeper_server} (return code: {process.returncode})")
     except subprocess.TimeoutExpired:
-        logger.warning(f"Timeout while connecting to {zookeeper_server}")
+        logger.warn(f"Timeout while connecting to {zookeeper_server}")
     except Exception as e:
         logger.exception(f"Error retrieving state information from {zookeeper_server}: {e}")
 
-    logger.warning(f"Returning 'NA' for server {zookeeper_server}")
+    logger.warn(f"Returning 'NA' for server {zookeeper_server}")
     return "NA"
 
 def get_leader_node(zookeeper_hosts):
@@ -143,9 +143,9 @@ def get_leader_node(zookeeper_hosts):
         logger.info(f"Unique leader found: {leaders[0]}")
         return leaders[0]  # Return just the hostname without port
     elif len(leaders) > 1:
-        logger.warning(f"Multiple leaders found: {leaders}. This is unexpected.")
+        logger.warn(f"Multiple leaders found: {leaders}. This is unexpected.")
     else:
-        logger.warning("No leader found among all hosts")
+        logger.warn("No leader found among all hosts")
 
     return "NA"