From 50e66df03cfd2d1e12c10fd83e2535b2c3ce5e44 Mon Sep 17 00:00:00 2001 From: kiryazovi-redis Date: Thu, 21 Aug 2025 16:48:13 +0300 Subject: [PATCH 1/3] Add fault injector client for Redis Enterprise testing This commit introduces a comprehensive fault injector client that provides a Python interface for interacting with Redis Enterprise fault injection services. The client supports: - Multiple action types (DMC restart, failover, reshard, network failure) - Direct rladmin command execution - Sequence of actions for complex testing scenarios - Full HTTP API integration with proper error handling - Debug output for troubleshooting test scenarios The fault injector client enables automated testing of Redis Enterprise cluster resilience and failover scenarios, supporting the CAE client testing framework approach for real enterprise server integration. --- tests/test_scenario/__init__.py | 0 tests/test_scenario/fault_injector_client.py | 108 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 tests/test_scenario/__init__.py create mode 100644 tests/test_scenario/fault_injector_client.py diff --git a/tests/test_scenario/__init__.py b/tests/test_scenario/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_scenario/fault_injector_client.py b/tests/test_scenario/fault_injector_client.py new file mode 100644 index 0000000000..915d914a3d --- /dev/null +++ b/tests/test_scenario/fault_injector_client.py @@ -0,0 +1,108 @@ +import json +import urllib.request +from typing import Dict, Any, Optional, Union +from enum import Enum + + +class ActionType(str, Enum): + DMC_RESTART = "dmc_restart" + FAILOVER = "failover" + RESHARD = "reshard" + SEQUENCE_OF_ACTIONS = "sequence_of_actions" + NETWORK_FAILURE = "network_failure" + EXECUTE_RLUTIL_COMMAND = "execute_rlutil_command" + EXECUTE_RLADMIN_COMMAND = "execute_rladmin_command" + + +class RestartDmcParams: + def __init__(self, bdb_id: str): + self.bdb_id = bdb_id + + def to_dict(self) -> Dict[str, str]: + return {"bdb_id": self.bdb_id} + + +class ActionRequest: + def __init__(self, action_type: ActionType, parameters: Union[Dict[str, Any], RestartDmcParams]): + self.type = action_type + self.parameters = parameters + + def to_dict(self) -> Dict[str, Any]: + return { + "type": self.type.value, # Use the string value of the enum + "parameters": self.parameters.to_dict() if isinstance(self.parameters, + RestartDmcParams) else self.parameters + } + + +class FaultInjectorClient: + def __init__(self, base_url: str): + self.base_url = base_url.rstrip('/') + + def _make_request(self, method: str, path: str, data: Optional[Dict] = None) -> Dict[str, Any]: + url = f"{self.base_url}{path}" + headers = {"Content-Type": "application/json"} if data else {} + + request_data = None + if data: + request_data = json.dumps(data).encode('utf-8') + print(f"JSON payload being sent: {request_data.decode('utf-8')}") + + request = urllib.request.Request( + url, + method=method, + data=request_data, + headers=headers + ) + + try: + with urllib.request.urlopen(request) as response: + return json.loads(response.read().decode('utf-8')) + except urllib.error.HTTPError as e: + if e.code == 422: + error_body = json.loads(e.read().decode('utf-8')) + raise ValueError(f"Validation Error: {error_body}") + raise + + def list_actions(self) -> Dict[str, Any]: + """List all available actions""" + return self._make_request("GET", "/action") + + def trigger_action(self, action_request: ActionRequest) -> Dict[str, Any]: + """Trigger a new action""" + request_data = action_request.to_dict() + print(f"Sending HTTP request data: {request_data}") + return self._make_request("POST", "/action", request_data) + + def get_action_status(self, action_id: str) -> Dict[str, Any]: + """Get the status of a specific action""" + return self._make_request("GET", f"/action/{action_id}") + + def execute_rladmin_command(self, command: str, bdb_id: str = None) -> Dict[str, Any]: + """Execute rladmin command directly as string""" + url = f"{self.base_url}/rladmin" + + # The fault injector expects the raw command string + command_string = f"rladmin {command}" + if bdb_id: + command_string = f"rladmin -b {bdb_id} {command}" + + print(f"Sending rladmin command: {command_string}") + + headers = {"Content-Type": "text/plain"} + + request = urllib.request.Request( + url, + method="POST", + data=command_string.encode('utf-8'), + headers=headers + ) + + try: + with urllib.request.urlopen(request) as response: + return json.loads(response.read().decode('utf-8')) + except urllib.error.HTTPError as e: + if e.code == 422: + error_body = json.loads(e.read().decode('utf-8')) + raise ValueError(f"Validation Error: {error_body}") + raise \ No newline at end of file From f6c6ef89d323f5bd2c5d72039bc61a93f81eda15 Mon Sep 17 00:00:00 2001 From: kiryazovi-redis Date: Thu, 21 Aug 2025 16:54:43 +0300 Subject: [PATCH 2/3] Apply linting fixes to fault injector client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fixed whitespace in blank lines (W293) - Added trailing newline at end of file (W292) - Applied ruff formatting for consistent code style: * Improved line breaks for long function signatures * Consistent quote style (single to double quotes) * Proper parameter alignment and indentation All linting checks now pass: - ruff check: ✅ All checks passed - ruff format: ✅ Code properly formatted - vulture: ✅ No dead code detected --- tests/test_scenario/fault_injector_client.py | 55 +++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/tests/test_scenario/fault_injector_client.py b/tests/test_scenario/fault_injector_client.py index 915d914a3d..efc44ac783 100644 --- a/tests/test_scenario/fault_injector_client.py +++ b/tests/test_scenario/fault_injector_client.py @@ -23,44 +23,48 @@ def to_dict(self) -> Dict[str, str]: class ActionRequest: - def __init__(self, action_type: ActionType, parameters: Union[Dict[str, Any], RestartDmcParams]): + def __init__( + self, + action_type: ActionType, + parameters: Union[Dict[str, Any], RestartDmcParams], + ): self.type = action_type self.parameters = parameters def to_dict(self) -> Dict[str, Any]: return { "type": self.type.value, # Use the string value of the enum - "parameters": self.parameters.to_dict() if isinstance(self.parameters, - RestartDmcParams) else self.parameters + "parameters": self.parameters.to_dict() + if isinstance(self.parameters, RestartDmcParams) + else self.parameters, } class FaultInjectorClient: def __init__(self, base_url: str): - self.base_url = base_url.rstrip('/') + self.base_url = base_url.rstrip("/") - def _make_request(self, method: str, path: str, data: Optional[Dict] = None) -> Dict[str, Any]: + def _make_request( + self, method: str, path: str, data: Optional[Dict] = None + ) -> Dict[str, Any]: url = f"{self.base_url}{path}" headers = {"Content-Type": "application/json"} if data else {} request_data = None if data: - request_data = json.dumps(data).encode('utf-8') + request_data = json.dumps(data).encode("utf-8") print(f"JSON payload being sent: {request_data.decode('utf-8')}") request = urllib.request.Request( - url, - method=method, - data=request_data, - headers=headers + url, method=method, data=request_data, headers=headers ) try: with urllib.request.urlopen(request) as response: - return json.loads(response.read().decode('utf-8')) + return json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as e: if e.code == 422: - error_body = json.loads(e.read().decode('utf-8')) + error_body = json.loads(e.read().decode("utf-8")) raise ValueError(f"Validation Error: {error_body}") raise @@ -77,32 +81,31 @@ def trigger_action(self, action_request: ActionRequest) -> Dict[str, Any]: def get_action_status(self, action_id: str) -> Dict[str, Any]: """Get the status of a specific action""" return self._make_request("GET", f"/action/{action_id}") - - def execute_rladmin_command(self, command: str, bdb_id: str = None) -> Dict[str, Any]: + + def execute_rladmin_command( + self, command: str, bdb_id: str = None + ) -> Dict[str, Any]: """Execute rladmin command directly as string""" url = f"{self.base_url}/rladmin" - + # The fault injector expects the raw command string command_string = f"rladmin {command}" if bdb_id: command_string = f"rladmin -b {bdb_id} {command}" - + print(f"Sending rladmin command: {command_string}") - + headers = {"Content-Type": "text/plain"} - + request = urllib.request.Request( - url, - method="POST", - data=command_string.encode('utf-8'), - headers=headers + url, method="POST", data=command_string.encode("utf-8"), headers=headers ) - + try: with urllib.request.urlopen(request) as response: - return json.loads(response.read().decode('utf-8')) + return json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as e: if e.code == 422: - error_body = json.loads(e.read().decode('utf-8')) + error_body = json.loads(e.read().decode("utf-8")) raise ValueError(f"Validation Error: {error_body}") - raise \ No newline at end of file + raise From 4b31c1c27a799a2c11c86792872f1b8e21dae516 Mon Sep 17 00:00:00 2001 From: kiryazovi-redis Date: Fri, 22 Aug 2025 18:48:34 +0300 Subject: [PATCH 3/3] take care of review issues --- tests/test_scenario/fault_injector_client.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_scenario/fault_injector_client.py b/tests/test_scenario/fault_injector_client.py index efc44ac783..9cb35b24b6 100644 --- a/tests/test_scenario/fault_injector_client.py +++ b/tests/test_scenario/fault_injector_client.py @@ -50,10 +50,7 @@ def _make_request( url = f"{self.base_url}{path}" headers = {"Content-Type": "application/json"} if data else {} - request_data = None - if data: - request_data = json.dumps(data).encode("utf-8") - print(f"JSON payload being sent: {request_data.decode('utf-8')}") + request_data = json.dumps(data).encode("utf-8") if data else None request = urllib.request.Request( url, method=method, data=request_data, headers=headers @@ -75,7 +72,6 @@ def list_actions(self) -> Dict[str, Any]: def trigger_action(self, action_request: ActionRequest) -> Dict[str, Any]: """Trigger a new action""" request_data = action_request.to_dict() - print(f"Sending HTTP request data: {request_data}") return self._make_request("POST", "/action", request_data) def get_action_status(self, action_id: str) -> Dict[str, Any]: @@ -93,8 +89,6 @@ def execute_rladmin_command( if bdb_id: command_string = f"rladmin -b {bdb_id} {command}" - print(f"Sending rladmin command: {command_string}") - headers = {"Content-Type": "text/plain"} request = urllib.request.Request(