Added lag tollerance parameter

Author: vladvildanov

redis/multidb/healthcheck.py

@@ -73,6 +73,7 @@ def __init__(
         self,
         retry=Retry(retries=DEFAULT_HEALTH_CHECK_RETRIES, backoff=DEFAULT_HEALTH_CHECK_BACKOFF),
         rest_api_port: int = 9443,
+        availability_lag_tolerance: int = 100,
         timeout: float = DEFAULT_TIMEOUT,
         auth_basic: Optional[Tuple[str, str]] = None,
         verify_tls: bool = True,
@@ -85,6 +86,24 @@ def __init__(
         client_key_file: Optional[str] = None,
         client_key_password: Optional[str] = None,
     ):
+        """
+        Initialize LagAwareHealthCheck with the specified parameters.
+
+        Args:
+            retry: Retry configuration for health checks
+            rest_api_port: Port number for Redis Enterprise REST API (default: 9443)
+            availability_lag_tolerance: Maximum acceptable lag in milliseconds (default: 100)
+            timeout: Request timeout in seconds (default: DEFAULT_TIMEOUT)
+            auth_basic: Tuple of (username, password) for basic authentication
+            verify_tls: Whether to verify TLS certificates (default: True)
+            ca_file: Path to CA certificate file for TLS verification
+            ca_path: Path to CA certificates directory for TLS verification
+            ca_data: CA certificate data as string or bytes
+            client_cert_file: Path to client certificate file for mutual TLS
+            client_key_file: Path to client private key file for mutual TLS
+            client_key_password: Password for encrypted client private key
+        """
+
         super().__init__(
             retry=retry,
         )
@@ -101,6 +120,7 @@ def __init__(
             client_key_password=client_key_password
         )
         self._rest_api_port = rest_api_port
+        self._availability_lag_tolerance = availability_lag_tolerance
 
     def check_health(self, database) -> bool:
         client = database.client
@@ -126,7 +146,7 @@ def check_health(self, database) -> bool:
             logger.warning("LagAwareHealthCheck failed: Couldn't find a matching bdb")
             raise ValueError("Could not find a matching bdb")
 
-        url = f"/v1/bdbs/{matching_bdb['uid']}/availability"
+        url = f"/v1/bdbs/{matching_bdb['uid']}/availability?availability_lag_tolerance_ms={self._availability_lag_tolerance}"
         self._http_client.get(url, expect_json=False)
 
         # Status checked in an http client, otherwise HttpError will be raised

tests/test_scenario/test_active_active.py

@@ -17,7 +17,7 @@ def trigger_network_failure_action(fault_injector_client, event: threading.Event
     endpoint_config = get_endpoint_config('re-active-active')
     action_request = ActionRequest(
         action_type=ActionType.NETWORK_FAILURE,
-        parameters={"bdb_id": endpoint_config['bdb_id'], "delay": 2, "cluster_index": 0}
+        parameters={"bdb_id": endpoint_config['bdb_id'], "delay": 3, "cluster_index": 0}
     )
 
     result = fault_injector_client.trigger_action(action_request)
@@ -37,7 +37,7 @@ class TestActiveActiveStandalone:
 
     def teardown_method(self, method):
         # Timeout so the cluster could recover from network failure.
-        sleep(4)
+        sleep(5)
 
     @pytest.mark.parametrize(
         "r_multi_db",
@@ -63,12 +63,12 @@ def test_multi_db_client_failover_to_another_db(self, r_multi_db, fault_injector
         # Execute commands before network failure
         while not event.is_set():
             assert r_multi_db.get('key') == 'value'
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute commands after network failure
         for _ in range(3):
             assert r_multi_db.get('key') == 'value'
-            sleep(0.1)
+            sleep(0.5)
 
         assert listener.is_changed_flag == True
 
@@ -104,12 +104,12 @@ def test_multi_db_client_uses_lag_aware_health_check(self, r_multi_db, fault_inj
         # Execute commands before network failure
         while not event.is_set():
             assert r_multi_db.get('key') == 'value'
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute commands after network failure
         for _ in range(3):
             assert r_multi_db.get('key') == 'value'
-            sleep(0.1)
+            sleep(0.5)
 
         assert listener.is_changed_flag == True
 
@@ -152,7 +152,7 @@ def test_context_manager_pipeline_failover_to_another_db(self, r_multi_db, fault
                 pipe.get('{hash}key2')
                 pipe.get('{hash}key3')
                 assert pipe.execute() == [True, True, True, 'value1', 'value2', 'value3']
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute pipeline after network failure
         for _ in range(3):
@@ -164,7 +164,7 @@ def test_context_manager_pipeline_failover_to_another_db(self, r_multi_db, fault
                 pipe.get('{hash}key2')
                 pipe.get('{hash}key3')
                 assert pipe.execute() == [True, True, True, 'value1', 'value2', 'value3']
-            sleep(0.1)
+            sleep(0.5)
 
         assert listener.is_changed_flag == True
 
@@ -206,7 +206,7 @@ def test_chaining_pipeline_failover_to_another_db(self, r_multi_db, fault_inject
             pipe.get('{hash}key2')
             pipe.get('{hash}key3')
             assert pipe.execute() == [True, True, True, 'value1', 'value2', 'value3']
-        sleep(0.1)
+        sleep(0.5)
 
         # Execute pipeline after network failure
         for _ in range(3):
@@ -217,7 +217,7 @@ def test_chaining_pipeline_failover_to_another_db(self, r_multi_db, fault_inject
             pipe.get('{hash}key2')
             pipe.get('{hash}key3')
             assert pipe.execute() == [True, True, True, 'value1', 'value2', 'value3']
-        sleep(0.1)
+        sleep(0.5)
 
         assert listener.is_changed_flag == True
 
@@ -253,12 +253,12 @@ def callback(pipe: Pipeline):
         # Execute pipeline before network failure
         while not event.is_set():
             r_multi_db.transaction(callback)
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute pipeline after network failure
         for _ in range(3):
             r_multi_db.transaction(callback)
-            sleep(0.1)
+            sleep(0.5)
 
         assert listener.is_changed_flag == True
 
@@ -295,12 +295,12 @@ def handler(message):
         # Execute pipeline before network failure
         while not event.is_set():
             r_multi_db.publish('test-channel', data)
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute pipeline after network failure
         for _ in range(3):
             r_multi_db.publish('test-channel', data)
-            sleep(0.1)
+            sleep(0.5)
 
         pubsub_thread.stop()
 
@@ -340,12 +340,12 @@ def handler(message):
         # Execute pipeline before network failure
         while not event.is_set():
             r_multi_db.spublish('test-channel', data)
-            sleep(0.1)
+            sleep(0.5)
 
         # Execute pipeline after network failure
         for _ in range(3):
             r_multi_db.spublish('test-channel', data)
-            sleep(0.1)
+            sleep(0.5)
 
         pubsub_thread.stop()