From 182cc7c11cca03ffc98b2ad39a307c16130f9bce Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Tue, 30 Sep 2025 10:24:22 +0300 Subject: [PATCH 01/44] update e2e test, change script --- maintnotifications/e2e/scenario_endpoint_types_test.go | 8 ++++++++ maintnotifications/e2e/scripts/run-e2e-tests.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index d1ff4f823..94a99f550 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -357,6 +357,14 @@ func TestEndpointTypesPushNotifications(t *testing.T) { e("Expected MOVING notifications with %s endpoint type, got none", endpointTest.name) } + logAnalysis := logCollector.GetAnalysis() + if logAnalysis.TotalHandoffCount == 0 { + e("Expected at least one handoff with %s endpoint type, got none", endpointTest.name) + } + if logAnalysis.TotalHandoffCount != logAnalysis.SucceededHandoffCount { + e("Expected all handoffs to succeed with %s endpoint type, got %d failed", endpointTest.name, logAnalysis.FailedHandoffCount) + } + if errorsDetected { logCollector.DumpLogs() trackerAnalysis.Print(t) diff --git a/maintnotifications/e2e/scripts/run-e2e-tests.sh b/maintnotifications/e2e/scripts/run-e2e-tests.sh index 9426fbddc..f1175523d 100755 --- a/maintnotifications/e2e/scripts/run-e2e-tests.sh +++ b/maintnotifications/e2e/scripts/run-e2e-tests.sh @@ -3,7 +3,7 @@ # Maintenance Notifications E2E Tests Runner # This script sets up the environment and runs the maintnotifications upgrade E2E tests -set -euo pipefail +set -eu # Script directory and repository root SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 460ea94322e64acbe6d05964d847671720a10f83 Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Tue, 30 Sep 2025 12:59:25 +0300 Subject: [PATCH 02/44] update script and tests --- maintnotifications/e2e/scenario_endpoint_types_test.go | 2 +- maintnotifications/e2e/scenario_push_notifications_test.go | 2 +- maintnotifications/e2e/scenario_stress_test.go | 2 +- maintnotifications/e2e/scenario_timeout_configs_test.go | 2 +- maintnotifications/e2e/scenario_tls_configs_test.go | 2 +- maintnotifications/e2e/scripts/run-e2e-tests.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 94a99f550..e6a31b20b 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -61,7 +61,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("m-standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) } diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 74d0a894b..0f8be88b0 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -59,7 +59,7 @@ func TestPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("m-standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping push notification tests: %v", err) } diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 5a788ef18..66fd1e37b 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -49,7 +49,7 @@ func TestStressPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("m-standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping stress test: %v", err) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 0477a53fb..054e967e9 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -77,7 +77,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("m-standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping timeout configs test: %v", err) } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index cbaec43a8..a6d8735cf 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -74,7 +74,7 @@ func TestTLSConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("m-standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping TLS configs test: %v", err) } diff --git a/maintnotifications/e2e/scripts/run-e2e-tests.sh b/maintnotifications/e2e/scripts/run-e2e-tests.sh index f1175523d..9426fbddc 100755 --- a/maintnotifications/e2e/scripts/run-e2e-tests.sh +++ b/maintnotifications/e2e/scripts/run-e2e-tests.sh @@ -3,7 +3,7 @@ # Maintenance Notifications E2E Tests Runner # This script sets up the environment and runs the maintnotifications upgrade E2E tests -set -eu +set -euo pipefail # Script directory and repository root SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 4287b8b7900b456bed5f1b1d93a1bc3715257d4e Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Tue, 30 Sep 2025 13:31:27 +0300 Subject: [PATCH 03/44] fixed bdbid parsing --- maintnotifications/e2e/config_parser_test.go | 14 ++++++++++++-- .../e2e/scenario_endpoint_types_test.go | 2 +- .../e2e/scenario_push_notifications_test.go | 2 +- maintnotifications/e2e/scenario_stress_test.go | 2 +- .../e2e/scenario_timeout_configs_test.go | 2 +- .../e2e/scenario_tls_configs_test.go | 2 +- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index e8e795a44..ea4901bfd 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -30,7 +30,7 @@ type DatabaseEndpoint struct { // DatabaseConfig represents the configuration for a single database type DatabaseConfig struct { - BdbID int `json:"bdb_id,omitempty"` + BdbID interface{} `json:"bdb_id,omitempty"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` TLS bool `json:"tls"` @@ -157,13 +157,23 @@ func GetDatabaseConfig(databasesConfig DatabasesConfig, databaseName string) (*R return nil, fmt.Errorf("no endpoints found in database configuration") } + var bdbId int + switch (dbConfig.BdbID).(type) { + case int: + bdbId = dbConfig.BdbID.(int) + case float64: + bdbId = int(dbConfig.BdbID.(float64)) + case string: + bdbId, _ = strconv.Atoi(dbConfig.BdbID.(string)) + } + return &RedisConnectionConfig{ Host: host, Port: port, Username: dbConfig.Username, Password: dbConfig.Password, TLS: dbConfig.TLS, - BdbID: dbConfig.BdbID, + BdbID: bdbId, CertificatesLocation: dbConfig.CertificatesLocation, Endpoints: dbConfig.Endpoints, }, nil diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index e6a31b20b..f884ee1b2 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -61,7 +61,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standalone") + factory, err := CreateTestClientFactory("m-standard") if err != nil { t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) } diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 0f8be88b0..8db0ccf35 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -59,7 +59,7 @@ func TestPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standalone") + factory, err := CreateTestClientFactory("m-standard") if err != nil { t.Skipf("Enterprise cluster not available, skipping push notification tests: %v", err) } diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 66fd1e37b..9ba0db530 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -49,7 +49,7 @@ func TestStressPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standalone") + factory, err := CreateTestClientFactory("m-standard") if err != nil { t.Skipf("Enterprise cluster not available, skipping stress test: %v", err) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 054e967e9..08fe2cf88 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -77,7 +77,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standalone") + factory, err := CreateTestClientFactory("m-standard") if err != nil { t.Skipf("Enterprise cluster not available, skipping timeout configs test: %v", err) } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index a6d8735cf..be50e3823 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -74,7 +74,7 @@ func TestTLSConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standalone") + factory, err := CreateTestClientFactory("m-standard") if err != nil { t.Skipf("Enterprise cluster not available, skipping TLS configs test: %v", err) } From a6133b35052e18aa1be7cc3793a3b113539e3f5f Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 11:01:47 +0300 Subject: [PATCH 04/44] disabled majority of tests, swapped event order --- .../e2e/scenario_endpoint_types_test.go | 128 +++++++++--------- .../e2e/scenario_push_notifications_test.go | 2 +- .../e2e/scenario_stress_test.go | 2 +- .../e2e/scenario_timeout_configs_test.go | 2 +- .../e2e/scenario_tls_configs_test.go | 2 +- 5 files changed, 68 insertions(+), 68 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index f884ee1b2..8d8fe083f 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -44,16 +44,16 @@ func TestEndpointTypesPushNotifications(t *testing.T) { endpointType: maintnotifications.EndpointTypeExternalIP, description: "External IP endpoint type for enterprise clusters", }, - { - name: "ExternalFQDN", - endpointType: maintnotifications.EndpointTypeExternalFQDN, - description: "External FQDN endpoint type for DNS-based routing", - }, - { - name: "None", - endpointType: maintnotifications.EndpointTypeNone, - description: "No endpoint type - reconnect with current config", - }, + // { + // name: "ExternalFQDN", + // endpointType: maintnotifications.EndpointTypeExternalFQDN, + // description: "External FQDN endpoint type for DNS-based routing", + // }, + // { + // name: "None", + // endpointType: maintnotifications.EndpointTypeNone, + // description: "No endpoint type - reconnect with current config", + // }, } defer func() { @@ -159,56 +159,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { commandsRunner.Stop() }() - // Test failover with this endpoint type - p("Testing failover with %s endpoint type...", endpointTest.name) - failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ - Type: "failover", - Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, - }, - }) - if err != nil { - t.Fatalf("Failed to trigger failover action for %s: %v", endpointTest.name, err) - } - - // Start command traffic - go func() { - commandsRunner.FireCommandsUntilStop(ctx) - }() - - // Wait for FAILING_OVER notification - match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 2*time.Minute) - if !found { - t.Fatalf("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) - } - failingOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILING_OVER notification received for %s. %v", endpointTest.name, failingOverData) - - // Wait for FAILED_OVER notification - seqIDToObserve := int64(failingOverData["seqID"].(float64)) - connIDToObserve := uint64(failingOverData["connID"].(float64)) - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) - if !found { - t.Fatalf("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) - } - failedOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) - - // Wait for failover to complete - status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), - ) - if err != nil { - t.Fatalf("[FI] Failover action failed for %s: %v", endpointTest.name, err) - } - p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) - // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ @@ -222,7 +172,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } // Wait for MIGRATING notification - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found := logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 30*time.Second) if !found { @@ -232,7 +182,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { p("MIGRATING notification received for %s: %v", endpointTest.name, migrateData) // Wait for migration to complete - status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, + status, err := faultInjector.WaitForAction(ctx, migrateResp.ActionID, WithMaxWaitTime(120*time.Second), WithPollInterval(1*time.Second), ) @@ -242,8 +192,8 @@ func TestEndpointTypesPushNotifications(t *testing.T) { p("[FI] Migrate action completed for %s: %s", endpointTest.name, status.Status) // Wait for MIGRATED notification - seqIDToObserve = int64(migrateData["seqID"].(float64)) - connIDToObserve = uint64(migrateData["connID"].(float64)) + seqIDToObserve := int64(migrateData["seqID"].(float64)) + connIDToObserve := uint64(migrateData["connID"].(float64)) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 2*time.Minute) @@ -326,6 +276,56 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } p("Bind action completed for %s: %s", endpointTest.name, bindStatus.Status) + // Test failover with this endpoint type + p("Testing failover with %s endpoint type...", endpointTest.name) + failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ + Type: "failover", + Parameters: map[string]interface{}{ + "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, + }, + }) + if err != nil { + t.Fatalf("Failed to trigger failover action for %s: %v", endpointTest.name, err) + } + + // Start command traffic + go func() { + commandsRunner.FireCommandsUntilStop(ctx) + }() + + // Wait for FAILING_OVER notification + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { + return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") + }, 2*time.Minute) + if !found { + t.Fatalf("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) + } + failingOverData := logs2.ExtractDataFromLogMessage(match) + p("FAILING_OVER notification received for %s. %v", endpointTest.name, failingOverData) + + // Wait for FAILED_OVER notification + seqIDToObserve = int64(failingOverData["seqID"].(float64)) + connIDToObserve = uint64(failingOverData["connID"].(float64)) + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { + return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) + }, 2*time.Minute) + if !found { + t.Fatalf("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) + } + failedOverData := logs2.ExtractDataFromLogMessage(match) + p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) + + // Wait for failover to complete + status, err = faultInjector.WaitForAction(ctx, failoverResp.ActionID, + WithMaxWaitTime(120*time.Second), + WithPollInterval(1*time.Second), + ) + if err != nil { + t.Fatalf("[FI] Failover action failed for %s: %v", endpointTest.name, err) + } + p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) + // Continue traffic for analysis time.Sleep(30 * time.Second) commandsRunner.Stop() diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 8db0ccf35..e548c3faa 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -14,7 +14,7 @@ import ( ) // TestPushNotifications tests Redis Enterprise push notifications (MOVING, MIGRATING, MIGRATED) -func TestPushNotifications(t *testing.T) { +func _TestPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 9ba0db530..604a06d52 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -14,7 +14,7 @@ import ( ) // TestStressPushNotifications tests push notifications under extreme stress conditions -func TestStressPushNotifications(t *testing.T) { +func _TestStressPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 08fe2cf88..831c05c68 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -14,7 +14,7 @@ import ( ) // TestTimeoutConfigurationsPushNotifications tests push notifications with different timeout configurations -func TestTimeoutConfigurationsPushNotifications(t *testing.T) { +func _TestTimeoutConfigurationsPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index be50e3823..fa18ba199 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -15,7 +15,7 @@ import ( // TODO ADD TLS CONFIGS // TestTLSConfigurationsPushNotifications tests push notifications with different TLS configurations -func TestTLSConfigurationsPushNotifications(t *testing.T) { +func _TestTLSConfigurationsPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } From 1fdbeb39cdac73f3166e11916dd9cb3410913b2a Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 11:40:55 +0300 Subject: [PATCH 05/44] change the config tag --- maintnotifications/e2e/scenario_endpoint_types_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 8d8fe083f..c200a27e2 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -61,7 +61,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standard") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) } From aa1b3b9199c3d3cdb42f9861091ad780f86f6eec Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 12:24:42 +0300 Subject: [PATCH 06/44] revert test order --- .../e2e/scenario_endpoint_types_test.go | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index c200a27e2..d0e04ba43 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -61,7 +61,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") + factory, err := CreateTestClientFactory("standlone") if err != nil { t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) } @@ -159,6 +159,56 @@ func TestEndpointTypesPushNotifications(t *testing.T) { commandsRunner.Stop() }() + // Test failover with this endpoint type + p("Testing failover with %s endpoint type...", endpointTest.name) + failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ + Type: "failover", + Parameters: map[string]interface{}{ + "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, + }, + }) + if err != nil { + t.Fatalf("Failed to trigger failover action for %s: %v", endpointTest.name, err) + } + + // Start command traffic + go func() { + commandsRunner.FireCommandsUntilStop(ctx) + }() + + // Wait for FAILING_OVER notification + match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { + return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") + }, 2*time.Minute) + if !found { + t.Fatalf("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) + } + failingOverData := logs2.ExtractDataFromLogMessage(match) + p("FAILING_OVER notification received for %s. %v", endpointTest.name, failingOverData) + + // Wait for FAILED_OVER notification + seqIDToObserve := int64(failingOverData["seqID"].(float64)) + connIDToObserve := uint64(failingOverData["connID"].(float64)) + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { + return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) + }, 2*time.Minute) + if !found { + t.Fatalf("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) + } + failedOverData := logs2.ExtractDataFromLogMessage(match) + p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) + + // Wait for failover to complete + status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, + WithMaxWaitTime(120*time.Second), + WithPollInterval(1*time.Second), + ) + if err != nil { + t.Fatalf("[FI] Failover action failed for %s: %v", endpointTest.name, err) + } + p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) + // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ @@ -172,7 +222,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } // Wait for MIGRATING notification - match, found := logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 30*time.Second) if !found { @@ -182,7 +232,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { p("MIGRATING notification received for %s: %v", endpointTest.name, migrateData) // Wait for migration to complete - status, err := faultInjector.WaitForAction(ctx, migrateResp.ActionID, + status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, WithMaxWaitTime(120*time.Second), WithPollInterval(1*time.Second), ) @@ -192,8 +242,8 @@ func TestEndpointTypesPushNotifications(t *testing.T) { p("[FI] Migrate action completed for %s: %s", endpointTest.name, status.Status) // Wait for MIGRATED notification - seqIDToObserve := int64(migrateData["seqID"].(float64)) - connIDToObserve := uint64(migrateData["connID"].(float64)) + seqIDToObserve = int64(migrateData["seqID"].(float64)) + connIDToObserve = uint64(migrateData["connID"].(float64)) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 2*time.Minute) @@ -276,56 +326,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } p("Bind action completed for %s: %s", endpointTest.name, bindStatus.Status) - // Test failover with this endpoint type - p("Testing failover with %s endpoint type...", endpointTest.name) - failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ - Type: "failover", - Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, - }, - }) - if err != nil { - t.Fatalf("Failed to trigger failover action for %s: %v", endpointTest.name, err) - } - - // Start command traffic - go func() { - commandsRunner.FireCommandsUntilStop(ctx) - }() - - // Wait for FAILING_OVER notification - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 2*time.Minute) - if !found { - t.Fatalf("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) - } - failingOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILING_OVER notification received for %s. %v", endpointTest.name, failingOverData) - - // Wait for FAILED_OVER notification - seqIDToObserve = int64(failingOverData["seqID"].(float64)) - connIDToObserve = uint64(failingOverData["connID"].(float64)) - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) - if !found { - t.Fatalf("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) - } - failedOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) - - // Wait for failover to complete - status, err = faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), - ) - if err != nil { - t.Fatalf("[FI] Failover action failed for %s: %v", endpointTest.name, err) - } - p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) - // Continue traffic for analysis time.Sleep(30 * time.Second) commandsRunner.Stop() From 36ad014de46fa027e37550d1556a310cfe557f87 Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 12:39:28 +0300 Subject: [PATCH 07/44] fix typo --- maintnotifications/e2e/scenario_endpoint_types_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index d0e04ba43..fe829c686 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -61,7 +61,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("standlone") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) } From 23ee78aeea15d179f541e9fec56735e9892b9299 Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 12:56:19 +0300 Subject: [PATCH 08/44] reenable all e2e tests --- .../e2e/scenario_endpoint_types_test.go | 20 +++++++++---------- .../e2e/scenario_push_notifications_test.go | 2 +- .../e2e/scenario_stress_test.go | 2 +- .../e2e/scenario_timeout_configs_test.go | 2 +- .../e2e/scenario_tls_configs_test.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index fe829c686..94a99f550 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -44,16 +44,16 @@ func TestEndpointTypesPushNotifications(t *testing.T) { endpointType: maintnotifications.EndpointTypeExternalIP, description: "External IP endpoint type for enterprise clusters", }, - // { - // name: "ExternalFQDN", - // endpointType: maintnotifications.EndpointTypeExternalFQDN, - // description: "External FQDN endpoint type for DNS-based routing", - // }, - // { - // name: "None", - // endpointType: maintnotifications.EndpointTypeNone, - // description: "No endpoint type - reconnect with current config", - // }, + { + name: "ExternalFQDN", + endpointType: maintnotifications.EndpointTypeExternalFQDN, + description: "External FQDN endpoint type for DNS-based routing", + }, + { + name: "None", + endpointType: maintnotifications.EndpointTypeNone, + description: "No endpoint type - reconnect with current config", + }, } defer func() { diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index e548c3faa..8db0ccf35 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -14,7 +14,7 @@ import ( ) // TestPushNotifications tests Redis Enterprise push notifications (MOVING, MIGRATING, MIGRATED) -func _TestPushNotifications(t *testing.T) { +func TestPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 604a06d52..9ba0db530 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -14,7 +14,7 @@ import ( ) // TestStressPushNotifications tests push notifications under extreme stress conditions -func _TestStressPushNotifications(t *testing.T) { +func TestStressPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 831c05c68..08fe2cf88 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -14,7 +14,7 @@ import ( ) // TestTimeoutConfigurationsPushNotifications tests push notifications with different timeout configurations -func _TestTimeoutConfigurationsPushNotifications(t *testing.T) { +func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index fa18ba199..a42326251 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -15,7 +15,7 @@ import ( // TODO ADD TLS CONFIGS // TestTLSConfigurationsPushNotifications tests push notifications with different TLS configurations -func _TestTLSConfigurationsPushNotifications(t *testing.T) { +func ТestTLSConfigurationsPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } From 7b098fa9a5c9a5af155102f60ba8a6bee968701b Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Thu, 2 Oct 2025 13:16:54 +0300 Subject: [PATCH 09/44] change the clonfig flag key for all e2e tests --- maintnotifications/e2e/scenario_push_notifications_test.go | 2 +- maintnotifications/e2e/scenario_stress_test.go | 2 +- maintnotifications/e2e/scenario_timeout_configs_test.go | 2 +- maintnotifications/e2e/scenario_tls_configs_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 8db0ccf35..74d0a894b 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -59,7 +59,7 @@ func TestPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standard") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping push notification tests: %v", err) } diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 9ba0db530..5a788ef18 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -49,7 +49,7 @@ func TestStressPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standard") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping stress test: %v", err) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 08fe2cf88..0477a53fb 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -77,7 +77,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standard") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping timeout configs test: %v", err) } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index a42326251..3dad2cc8e 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -74,7 +74,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { }() // Create client factory from configuration - factory, err := CreateTestClientFactory("m-standard") + factory, err := CreateTestClientFactory("standalone") if err != nil { t.Skipf("Enterprise cluster not available, skipping TLS configs test: %v", err) } From 72cf03288d403c15d493d60c2da27b1dfa87da61 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 16:04:14 +0300 Subject: [PATCH 10/44] improve logging for debug purposes of tests --- .../e2e/scenario_endpoint_types_test.go | 39 ++++++++++------ .../e2e/scenario_push_notifications_test.go | 46 +++++++++++-------- .../e2e/scenario_stress_test.go | 27 ++++++++--- .../e2e/scenario_timeout_configs_test.go | 44 ++++++++++-------- .../e2e/scenario_tls_configs_test.go | 33 +++++++------ 5 files changed, 117 insertions(+), 72 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 94a99f550..31613c953 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -70,7 +70,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Create fault injector faultInjector, err := CreateTestFaultInjector() if err != nil { - t.Fatalf("Failed to create fault injector: %v", err) + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) } defer func() { @@ -103,6 +103,15 @@ func TestEndpointTypesPushNotifications(t *testing.T) { args = append([]interface{}{ts}, args...) t.Errorf(format, args...) } + + var ef = func(format string, args ...interface{}) { + errorsDetected = true + format = "[%s][ENDPOINT-TYPES][ERROR] " + format + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{ts}, args...) + t.Fatalf(format, args...) + } + p("Testing endpoint type: %s - %s", endpointTest.name, endpointTest.description) minIdleConns := 3 @@ -126,7 +135,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { ClientName: fmt.Sprintf("endpoint-test-%s", endpointTest.name), }) if err != nil { - t.Fatalf("Failed to create client for %s: %v", endpointTest.name, err) + ef("Failed to create client for %s: %v", endpointTest.name, err) } // Create timeout tracker @@ -144,7 +153,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Verify initial connectivity err = client.Ping(ctx).Err() if err != nil { - t.Fatalf("Failed to ping Redis with %s endpoint type: %v", endpointTest.name, err) + ef("Failed to ping Redis with %s endpoint type: %v", endpointTest.name, err) } p("Client connected successfully with %s endpoint type", endpointTest.name) @@ -169,7 +178,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger failover action for %s: %v", endpointTest.name, err) + ef("Failed to trigger failover action for %s: %v", endpointTest.name, err) } // Start command traffic @@ -182,7 +191,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") }, 2*time.Minute) if !found { - t.Fatalf("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) + ef("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) } failingOverData := logs2.ExtractDataFromLogMessage(match) p("FAILING_OVER notification received for %s. %v", endpointTest.name, failingOverData) @@ -194,7 +203,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 2*time.Minute) if !found { - t.Fatalf("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) + ef("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) } failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) @@ -205,7 +214,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Failover action failed for %s: %v", endpointTest.name, err) + ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) } p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) @@ -218,7 +227,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger migrate action for %s: %v", endpointTest.name, err) + ef("Failed to trigger migrate action for %s: %v", endpointTest.name, err) } // Wait for MIGRATING notification @@ -226,7 +235,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 30*time.Second) if !found { - t.Fatalf("MIGRATING notification was not received for %s endpoint type", endpointTest.name) + ef("MIGRATING notification was not received for %s endpoint type", endpointTest.name) } migrateData := logs2.ExtractDataFromLogMessage(match) p("MIGRATING notification received for %s: %v", endpointTest.name, migrateData) @@ -237,7 +246,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Migrate action failed for %s: %v", endpointTest.name, err) + ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) } p("[FI] Migrate action completed for %s: %s", endpointTest.name, status.Status) @@ -248,7 +257,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 2*time.Minute) if !found { - t.Fatalf("MIGRATED notification was not received for %s endpoint type", endpointTest.name) + ef("MIGRATED notification was not received for %s endpoint type", endpointTest.name) } migratedData := logs2.ExtractDataFromLogMessage(match) p("MIGRATED notification received for %s. %v", endpointTest.name, migratedData) @@ -262,7 +271,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger bind action for %s: %v", endpointTest.name, err) + ef("Failed to trigger bind action for %s: %v", endpointTest.name, err) } // Wait for MOVING notification @@ -270,7 +279,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") }, 2*time.Minute) if !found { - t.Fatalf("MOVING notification was not received for %s endpoint type", endpointTest.name) + ef("MOVING notification was not received for %s endpoint type", endpointTest.name) } movingData := logs2.ExtractDataFromLogMessage(match) p("MOVING notification received for %s. %v", endpointTest.name, movingData) @@ -322,7 +331,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { WithMaxWaitTime(120*time.Second), WithPollInterval(2*time.Second)) if err != nil { - t.Fatalf("Bind action failed for %s: %v", endpointTest.name, err) + ef("Bind action failed for %s: %v", endpointTest.name, err) } p("Bind action completed for %s: %s", endpointTest.name, bindStatus.Status) @@ -370,7 +379,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { trackerAnalysis.Print(t) logCollector.Clear() tracker.Clear() - t.Fatalf("[FAIL] Errors detected with %s endpoint type", endpointTest.name) + ef("[FAIL] Errors detected with %s endpoint type", endpointTest.name) } dump = false p("Endpoint type %s test completed successfully", endpointTest.name) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 74d0a894b..ead2f8a30 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -32,7 +32,7 @@ func TestPushNotifications(t *testing.T) { var status *ActionStatusResponse var p = func(format string, args ...interface{}) { - format = "[%s] " + format + format = "[%s][PUSH-NOTIFICATIONS] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Logf(format, args...) @@ -41,12 +41,20 @@ func TestPushNotifications(t *testing.T) { var errorsDetected = false var e = func(format string, args ...interface{}) { errorsDetected = true - format = "[%s][ERROR] " + format + format = "[%s][PUSH-NOTIFICATIONS][ERROR] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Errorf(format, args...) } + var ef = func(format string, args ...interface{}) { + errorsDetected = true + format = "[%s][PUSH-NOTIFICATIONS][ERROR] " + format + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{ts}, args...) + t.Fatalf(format, args...) + } + logCollector.ClearLogs() defer func() { if dump { @@ -61,14 +69,14 @@ func TestPushNotifications(t *testing.T) { // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { - t.Skipf("Enterprise cluster not available, skipping push notification tests: %v", err) + t.Skipf("[PUSH-NOTIFICATIONS][SKIP] Enterprise cluster not available, skipping push notification tests: %v", err) } endpointConfig := factory.GetConfig() // Create fault injector faultInjector, err := CreateTestFaultInjector() if err != nil { - t.Fatalf("Failed to create fault injector: %v", err) + ef("Failed to create fault injector: %v", err) } minIdleConns := 5 @@ -91,7 +99,7 @@ func TestPushNotifications(t *testing.T) { ClientName: "push-notification-test-client", }) if err != nil { - t.Fatalf("Failed to create client: %v", err) + ef("Failed to create client: %v", err) } defer func() { @@ -116,7 +124,7 @@ func TestPushNotifications(t *testing.T) { // Verify initial connectivity err = client.Ping(ctx).Err() if err != nil { - t.Fatalf("Failed to ping Redis: %v", err) + ef("Failed to ping Redis: %v", err) } p("Client connected successfully, starting push notification test") @@ -143,7 +151,7 @@ func TestPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger failover action: %v", err) + ef("Failed to trigger failover action: %v", err) } go func() { p("Waiting for FAILING_OVER notification") @@ -154,7 +162,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - t.Fatal("FAILING_OVER notification was not received within 2 minutes") + ef("FAILING_OVER notification was not received within 2 minutes") } failingOverData := logs2.ExtractDataFromLogMessage(match) p("FAILING_OVER notification received. %v", failingOverData) @@ -169,7 +177,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - t.Fatal("FAILED_OVER notification was not received within 2 minutes") + ef("FAILED_OVER notification was not received within 2 minutes") } failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received. %v", failedOverData) @@ -179,7 +187,7 @@ func TestPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Failover action failed: %v", err) + ef("[FI] Failover action failed: %v", err) } fmt.Printf("[FI] Failover action completed: %s\n", status.Status) @@ -194,7 +202,7 @@ func TestPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger migrate action: %v", err) + ef("Failed to trigger migrate action: %v", err) } go func() { match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { @@ -204,7 +212,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - t.Fatal("MIGRATING notification for migrate action was not received within 20 seconds") + ef("MIGRATING notification for migrate action was not received within 20 seconds") } migrateData := logs2.ExtractDataFromLogMessage(match) seqIDToObserve = int64(migrateData["seqID"].(float64)) @@ -216,7 +224,7 @@ func TestPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Migrate action failed: %v", err) + ef("[FI] Migrate action failed: %v", err) } fmt.Printf("[FI] Migrate action completed: %s\n", status.Status) @@ -229,7 +237,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - t.Fatal("MIGRATED notification was not received within 2 minutes") + ef("MIGRATED notification was not received within 2 minutes") } migratedData := logs2.ExtractDataFromLogMessage(match) p("MIGRATED notification received. %v", migratedData) @@ -247,7 +255,7 @@ func TestPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger bind action: %v", err) + ef("Failed to trigger bind action: %v", err) } // start a second client but don't execute any commands on it @@ -269,7 +277,7 @@ func TestPushNotifications(t *testing.T) { }) if err != nil { - t.Fatalf("failed to create client: %v", err) + ef("failed to create client: %v", err) } // setup tracking for second client tracker2 := NewTrackingNotificationsHook() @@ -336,7 +344,7 @@ func TestPushNotifications(t *testing.T) { // Wait for the goroutine to complete and check for errors if err := <-errChan; err != nil { - t.Fatalf("Second client goroutine error: %v", err) + ef("Second client goroutine error: %v", err) } // Wait for bind action to complete @@ -344,7 +352,7 @@ func TestPushNotifications(t *testing.T) { WithMaxWaitTime(120*time.Second), WithPollInterval(2*time.Second)) if err != nil { - t.Fatalf("Bind action failed: %v", err) + ef("Bind action failed: %v", err) } p("Bind action completed: %s", bindStatus.Status) @@ -457,7 +465,7 @@ func TestPushNotifications(t *testing.T) { trackerAnalysis.Print(t) logCollector.Clear() tracker.Clear() - t.Fatalf("[FAIL] Errors detected in push notification test") + ef("[FAIL] Errors detected in push notification test") } p("Analysis complete, no errors found") diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 5a788ef18..94d3ce7f7 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -16,7 +16,7 @@ import ( // TestStressPushNotifications tests push notifications under extreme stress conditions func TestStressPushNotifications(t *testing.T) { if os.Getenv("E2E_SCENARIO_TESTS") != "true" { - t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") + t.Skip("[STRESS][SKIP] Scenario tests require E2E_SCENARIO_TESTS=true") } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) @@ -30,12 +30,21 @@ func TestStressPushNotifications(t *testing.T) { t.Logf(format, args...) } + var errorsDetected = false var e = func(format string, args ...interface{}) { + errorsDetected = true format = "[%s][STRESS][ERROR] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Errorf(format, args...) } + var ef = func(format string, args ...interface{}) { + errorsDetected = true + format = "[%s][STRESS][ERROR] " + format + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{ts}, args...) + t.Fatalf(format, args...) + } logCollector.ClearLogs() defer func() { @@ -51,14 +60,14 @@ func TestStressPushNotifications(t *testing.T) { // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { - t.Skipf("Enterprise cluster not available, skipping stress test: %v", err) + t.Skipf("[STRESS][SKIP] Enterprise cluster not available, skipping stress test: %v", err) } endpointConfig := factory.GetConfig() // Create fault injector faultInjector, err := CreateTestFaultInjector() if err != nil { - t.Fatalf("Failed to create fault injector: %v", err) + ef("Failed to create fault injector: %v", err) } // Extreme stress configuration @@ -90,7 +99,7 @@ func TestStressPushNotifications(t *testing.T) { ClientName: fmt.Sprintf("stress-test-client-%d", i), }) if err != nil { - t.Fatalf("Failed to create stress client %d: %v", i, err) + ef("Failed to create stress client %d: %v", i, err) } clients = append(clients, client) @@ -124,7 +133,7 @@ func TestStressPushNotifications(t *testing.T) { for i, client := range clients { err = client.Ping(ctx).Err() if err != nil { - t.Fatalf("Failed to ping Redis with stress client %d: %v", i, err) + ef("Failed to ping Redis with stress client %d: %v", i, err) } } @@ -287,14 +296,18 @@ func TestStressPushNotifications(t *testing.T) { e("Too many notification processing errors under stress: %d/%d", totalProcessingErrors, totalTrackerNotifications) } - p("Stress test completed successfully!") + if errorsDetected { + ef("Errors detected under stress") + } + + dump = false + p("[SUCCESS] Stress test completed successfully!") p("Processed %d operations across %d clients with %d connections", totalOperations, numClients, allLogsAnalysis.ConnectionCount) p("Error rate: %.2f%%, Notification processing errors: %d/%d", errorRate, totalProcessingErrors, totalTrackerNotifications) // Print final analysis - dump = false allLogsAnalysis.Print(t) for i, tracker := range trackers { p("=== Stress Client %d Analysis ===", i) diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 0477a53fb..7bee18aa7 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -42,8 +42,8 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { { name: "Conservative", handoffTimeout: 60 * time.Second, - relaxedTimeout: 20 * time.Second, - postHandoffRelaxedDuration: 5 * time.Second, + relaxedTimeout: 60 * time.Second, + postHandoffRelaxedDuration: 2 * time.Minute, description: "Conservative timeouts for stable environments", expectedBehavior: "Longer timeouts, fewer timeout errors", }, @@ -79,14 +79,14 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { - t.Skipf("Enterprise cluster not available, skipping timeout configs test: %v", err) + t.Skipf("[TIMEOUT-CONFIGS][SKIP] Enterprise cluster not available, skipping timeout configs test: %v", err) } endpointConfig := factory.GetConfig() // Create fault injector faultInjector, err := CreateTestFaultInjector() if err != nil { - t.Fatalf("Failed to create fault injector: %v", err) + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) } defer func() { @@ -103,18 +103,26 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // redefine p and e for each test to get // proper test name in logs and proper test failures var p = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES] " + format + format = "[%s][TIMEOUT-CONFIGS] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Logf(format, args...) } var e = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES][ERROR] " + format + format = "[%s][TIMEOUT-CONFIGS][ERROR] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Errorf(format, args...) } + + var ef = func(format string, args ...interface{}) { + format = "[%s][TIMEOUT-CONFIGS][ERROR] " + format + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{ts}, args...) + t.Fatalf(format, args...) + } + p("Testing timeout configuration: %s - %s", timeoutTest.name, timeoutTest.description) p("Expected behavior: %s", timeoutTest.expectedBehavior) p("Handoff timeout: %v, Relaxed timeout: %v, Post-handoff duration: %v", @@ -141,7 +149,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ClientName: fmt.Sprintf("timeout-test-%s", timeoutTest.name), }) if err != nil { - t.Fatalf("Failed to create client for %s: %v", timeoutTest.name, err) + ef("Failed to create client for %s: %v", timeoutTest.name, err) } // Create timeout tracker @@ -159,7 +167,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Verify initial connectivity err = client.Ping(ctx).Err() if err != nil { - t.Fatalf("Failed to ping Redis with %s timeout config: %v", timeoutTest.name, err) + ef("Failed to ping Redis with %s timeout config: %v", timeoutTest.name, err) } p("Client connected successfully with %s timeout configuration", timeoutTest.name) @@ -192,7 +200,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger failover action for %s: %v", timeoutTest.name, err) + ef("Failed to trigger failover action for %s: %v", timeoutTest.name, err) } // Wait for FAILING_OVER notification @@ -200,7 +208,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") }, 3*time.Minute) if !found { - t.Fatalf("FAILING_OVER notification was not received for %s timeout config", timeoutTest.name) + ef("FAILING_OVER notification was not received for %s timeout config", timeoutTest.name) } failingOverData := logs2.ExtractDataFromLogMessage(match) p("FAILING_OVER notification received for %s. %v", timeoutTest.name, failingOverData) @@ -212,7 +220,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 3*time.Minute) if !found { - t.Fatalf("FAILED_OVER notification was not received for %s timeout config", timeoutTest.name) + ef("FAILED_OVER notification was not received for %s timeout config", timeoutTest.name) } failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received for %s. %v", timeoutTest.name, failedOverData) @@ -223,7 +231,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Failover action failed for %s: %v", timeoutTest.name, err) + ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err) } p("[FI] Failover action completed for %s: %s", timeoutTest.name, status.Status) @@ -240,7 +248,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger migrate action for %s: %v", timeoutTest.name, err) + ef("Failed to trigger migrate action for %s: %v", timeoutTest.name, err) } // Wait for MIGRATING notification @@ -248,7 +256,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 30*time.Second) if !found { - t.Fatalf("MIGRATING notification was not received for %s timeout config", timeoutTest.name) + ef("MIGRATING notification was not received for %s timeout config", timeoutTest.name) } migrateData := logs2.ExtractDataFromLogMessage(match) p("MIGRATING notification received for %s: %v", timeoutTest.name, migrateData) @@ -259,7 +267,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) + ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) } p("[FI] Migrate action completed for %s: %s", timeoutTest.name, status.Status) @@ -272,14 +280,14 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger bind action for %s: %v", timeoutTest.name, err) + ef("Failed to trigger bind action for %s: %v", timeoutTest.name, err) } status, err = faultInjector.WaitForAction(ctx, bindResp.ActionID, WithMaxWaitTime(120*time.Second), WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Bind action failed for %s: %v", timeoutTest.name, err) + ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err) } p("[FI] Bind action completed for %s: %s", timeoutTest.name, status.Status) // waiting for moving notification @@ -287,7 +295,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") }, 2*time.Minute) if !found { - t.Fatalf("MOVING notification was not received for %s timeout config", timeoutTest.name) + ef("MOVING notification was not received for %s timeout config", timeoutTest.name) } movingData := logs2.ExtractDataFromLogMessage(match) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 3dad2cc8e..6875d3f56 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -76,14 +76,14 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { - t.Skipf("Enterprise cluster not available, skipping TLS configs test: %v", err) + t.Skipf("[TLS-CONFIGS][SKIP] Enterprise cluster not available, skipping TLS configs test: %v", err) } endpointConfig := factory.GetConfig() // Create fault injector faultInjector, err := CreateTestFaultInjector() if err != nil { - t.Fatalf("Failed to create fault injector: %v", err) + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) } defer func() { @@ -100,18 +100,25 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // redefine p and e for each test to get // proper test name in logs and proper test failures var p = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES] " + format + format = "[%s][TLS-CONFIGS] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Logf(format, args...) } var e = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES][ERROR] " + format + format = "[%s][TLS-CONFIGS][ERROR] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Errorf(format, args...) } + + var ef = func(format string, args ...interface{}) { + format = "[%s][TLS-CONFIGS][ERROR] " + format + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{ts}, args...) + t.Fatalf(format, args...) + } if tlsTest.skipReason != "" { t.Skipf("Skipping %s: %s", tlsTest.name, tlsTest.skipReason) } @@ -144,7 +151,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { if tlsTest.name == "TLSSecure" || tlsTest.name == "TLSStrict" { t.Skipf("TLS configuration %s failed (expected in test environment): %v", tlsTest.name, err) } - t.Fatalf("Failed to create client for %s: %v", tlsTest.name, err) + ef("Failed to create client for %s: %v", tlsTest.name, err) } // Create timeout tracker @@ -165,7 +172,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { if tlsTest.name == "TLSSecure" || tlsTest.name == "TLSStrict" { t.Skipf("TLS configuration %s ping failed (expected in test environment): %v", tlsTest.name, err) } - t.Fatalf("Failed to ping Redis with %s TLS config: %v", tlsTest.name, err) + ef("Failed to ping Redis with %s TLS config: %v", tlsTest.name, err) } p("Client connected successfully with %s TLS configuration", tlsTest.name) @@ -195,7 +202,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger failover action for %s: %v", tlsTest.name, err) + ef("Failed to trigger failover action for %s: %v", tlsTest.name, err) } // Wait for FAILING_OVER notification @@ -203,7 +210,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") }, 2*time.Minute) if !found { - t.Fatalf("FAILING_OVER notification was not received for %s TLS config", tlsTest.name) + ef("FAILING_OVER notification was not received for %s TLS config", tlsTest.name) } failingOverData := logs2.ExtractDataFromLogMessage(match) p("FAILING_OVER notification received for %s. %v", tlsTest.name, failingOverData) @@ -215,7 +222,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) }, 2*time.Minute) if !found { - t.Fatalf("FAILED_OVER notification was not received for %s TLS config", tlsTest.name) + ef("FAILED_OVER notification was not received for %s TLS config", tlsTest.name) } failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received for %s. %v", tlsTest.name, failedOverData) @@ -226,7 +233,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Failover action failed for %s: %v", tlsTest.name, err) + ef("[FI] Failover action failed for %s: %v", tlsTest.name, err) } p("[FI] Failover action completed for %s: %s", tlsTest.name, status.Status) @@ -239,7 +246,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { }, }) if err != nil { - t.Fatalf("Failed to trigger migrate action for %s: %v", tlsTest.name, err) + ef("Failed to trigger migrate action for %s: %v", tlsTest.name, err) } // Wait for MIGRATING notification @@ -247,7 +254,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 30*time.Second) if !found { - t.Fatalf("MIGRATING notification was not received for %s TLS config", tlsTest.name) + ef("MIGRATING notification was not received for %s TLS config", tlsTest.name) } migrateData := logs2.ExtractDataFromLogMessage(match) p("MIGRATING notification received for %s: %v", tlsTest.name, migrateData) @@ -258,7 +265,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { WithPollInterval(1*time.Second), ) if err != nil { - t.Fatalf("[FI] Migrate action failed for %s: %v", tlsTest.name, err) + ef("[FI] Migrate action failed for %s: %v", tlsTest.name, err) } p("[FI] Migrate action completed for %s: %s", tlsTest.name, status.Status) From 75ae204907bf1abfb247ca107ad62a562762397d Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 16:38:35 +0300 Subject: [PATCH 11/44] longer deadline for FI in CI --- maintnotifications/e2e/scenario_endpoint_types_test.go | 10 +++++----- .../e2e/scenario_push_notifications_test.go | 10 +++++----- maintnotifications/e2e/scenario_stress_test.go | 2 +- maintnotifications/e2e/scenario_template.go.example | 2 +- .../e2e/scenario_timeout_configs_test.go | 10 +++++----- maintnotifications/e2e/scenario_tls_configs_test.go | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 31613c953..9e7a6c832 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -210,8 +210,8 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for failover to complete status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) @@ -242,8 +242,8 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for migration to complete status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) @@ -328,7 +328,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for bind to complete bindStatus, err := faultInjector.WaitForAction(ctx, bindResp.ActionID, - WithMaxWaitTime(120*time.Second), + WithMaxWaitTime(240*time.Second), WithPollInterval(2*time.Second)) if err != nil { ef("Bind action failed for %s: %v", endpointTest.name, err) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index ead2f8a30..9ce0485a7 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -183,8 +183,8 @@ func TestPushNotifications(t *testing.T) { p("FAILED_OVER notification received. %v", failedOverData) status, err = faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Failover action failed: %v", err) @@ -220,8 +220,8 @@ func TestPushNotifications(t *testing.T) { p("MIGRATING notification received: seqID: %d, connID: %d", seqIDToObserve, connIDToObserve) status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Migrate action failed: %v", err) @@ -349,7 +349,7 @@ func TestPushNotifications(t *testing.T) { // Wait for bind action to complete bindStatus, err := faultInjector.WaitForAction(ctx, bindResp.ActionID, - WithMaxWaitTime(120*time.Second), + WithMaxWaitTime(240*time.Second), WithPollInterval(2*time.Second)) if err != nil { ef("Bind action failed: %v", err) diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 94d3ce7f7..b2358e00e 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -208,7 +208,7 @@ func TestStressPushNotifications(t *testing.T) { // Wait for action to complete status, err := faultInjector.WaitForAction(ctx, resp.ActionID, - WithMaxWaitTime(300*time.Second), // Very long wait for stress + WithMaxWaitTime(360*time.Second), // Longer wait time for stress WithPollInterval(2*time.Second), ) if err != nil { diff --git a/maintnotifications/e2e/scenario_template.go.example b/maintnotifications/e2e/scenario_template.go.example index 963971502..50791aa65 100644 --- a/maintnotifications/e2e/scenario_template.go.example +++ b/maintnotifications/e2e/scenario_template.go.example @@ -130,7 +130,7 @@ func TestScenarioTemplate(t *testing.T) { // Step 8: Wait for fault injection to complete // status, err := faultInjector.WaitForAction(ctx, resp.ActionID, - // WithMaxWaitTime(120*time.Second), + // WithMaxWaitTime(240*time.Second), // WithPollInterval(2*time.Second)) // if err != nil { // t.Fatalf("Fault injection failed: %v", err) diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 7bee18aa7..aefbc2ec3 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -228,7 +228,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Wait for failover to complete status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, WithMaxWaitTime(180*time.Second), - WithPollInterval(1*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err) @@ -263,8 +263,8 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Wait for migration to complete status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) @@ -283,8 +283,8 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ef("Failed to trigger bind action for %s: %v", timeoutTest.name, err) } status, err = faultInjector.WaitForAction(ctx, bindResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 6875d3f56..d581ef787 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -229,8 +229,8 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Wait for failover to complete status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Failover action failed for %s: %v", tlsTest.name, err) @@ -261,8 +261,8 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Wait for migration to complete status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, - WithMaxWaitTime(120*time.Second), - WithPollInterval(1*time.Second), + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), ) if err != nil { ef("[FI] Migrate action failed for %s: %v", tlsTest.name, err) From 87670b265336ff2da7f834f8206e175ac13c7001 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 16:44:13 +0300 Subject: [PATCH 12/44] increase waiting for notifications --- .../e2e/scenario_endpoint_types_test.go | 8 ++++---- .../e2e/scenario_push_notifications_test.go | 12 ++++++------ .../e2e/scenario_timeout_configs_test.go | 4 ++-- maintnotifications/e2e/scenario_tls_configs_test.go | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 9e7a6c832..c4799a2b0 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -189,7 +189,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for FAILING_OVER notification match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("FAILING_OVER notification was not received for %s endpoint type", endpointTest.name) } @@ -201,7 +201,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { connIDToObserve := uint64(failingOverData["connID"].(float64)) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("FAILED_OVER notification was not received for %s endpoint type", endpointTest.name) } @@ -255,7 +255,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { connIDToObserve = uint64(migrateData["connID"].(float64)) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("MIGRATED notification was not received for %s endpoint type", endpointTest.name) } @@ -277,7 +277,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for MOVING notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("MOVING notification was not received for %s endpoint type", endpointTest.name) } diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 9ce0485a7..65c54a599 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -157,7 +157,7 @@ func TestPushNotifications(t *testing.T) { p("Waiting for FAILING_OVER notification") match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 2*time.Minute) + }, 3*time.Minute) commandsRunner.Stop() }() commandsRunner.FireCommandsUntilStop(ctx) @@ -172,7 +172,7 @@ func TestPushNotifications(t *testing.T) { p("Waiting for FAILED_OVER notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) + }, 3*time.Minute) commandsRunner.Stop() }() commandsRunner.FireCommandsUntilStop(ctx) @@ -232,7 +232,7 @@ func TestPushNotifications(t *testing.T) { p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) + }, 3*time.Minute) commandsRunner.Stop() }() commandsRunner.FireCommandsUntilStop(ctx) @@ -299,7 +299,7 @@ func TestPushNotifications(t *testing.T) { p("Waiting for MOVING notification on second client") match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") - }, 2*time.Minute) + }, 3*time.Minute) commandsRunner.Stop() // once moving is received, start a second client commands runner p("Starting commands on second client") @@ -315,7 +315,7 @@ func TestPushNotifications(t *testing.T) { // also validate big enough relaxed timeout match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 18) - }, 2*time.Minute) + }, 3*time.Minute) if !found { errChan <- fmt.Errorf("MOVING notification was not received within 2 minutes ON A SECOND CLIENT") return @@ -325,7 +325,7 @@ func TestPushNotifications(t *testing.T) { // wait for relaxation of 30m match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ApplyingRelaxedTimeoutDueToPostHandoffMessage) && strings.Contains(s, "30m") - }, 2*time.Minute) + }, 3*time.Minute) if !found { errChan <- fmt.Errorf("relaxed timeout was not applied within 2 minutes ON A SECOND CLIENT") return diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index aefbc2ec3..f3f522ea0 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -42,7 +42,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { { name: "Conservative", handoffTimeout: 60 * time.Second, - relaxedTimeout: 60 * time.Second, + relaxedTimeout: 30 * time.Second, postHandoffRelaxedDuration: 2 * time.Minute, description: "Conservative timeouts for stable environments", expectedBehavior: "Longer timeouts, fewer timeout errors", @@ -293,7 +293,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // waiting for moving notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("MOVING notification was not received for %s timeout config", timeoutTest.name) } diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index d581ef787..b1dc79181 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -208,7 +208,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Wait for FAILING_OVER notification match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("FAILING_OVER notification was not received for %s TLS config", tlsTest.name) } @@ -220,7 +220,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { connIDToObserve := uint64(failingOverData["connID"].(float64)) match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 2*time.Minute) + }, 3*time.Minute) if !found { ef("FAILED_OVER notification was not received for %s TLS config", tlsTest.name) } From 113edc59b54f5801e0bbbb94163df3ec6eec1f90 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 16:56:06 +0300 Subject: [PATCH 13/44] extend tests --- .../e2e/scenario_push_notifications_test.go | 68 +++++++++++++++++-- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 65c54a599..d839b902f 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strconv" "strings" "testing" "time" @@ -162,7 +163,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - ef("FAILING_OVER notification was not received within 2 minutes") + ef("FAILING_OVER notification was not received within 3 minutes") } failingOverData := logs2.ExtractDataFromLogMessage(match) p("FAILING_OVER notification received. %v", failingOverData) @@ -177,7 +178,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - ef("FAILED_OVER notification was not received within 2 minutes") + ef("FAILED_OVER notification was not received within 3 minutes") } failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received. %v", failedOverData) @@ -237,7 +238,7 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - ef("MIGRATED notification was not received within 2 minutes") + ef("MIGRATED notification was not received within 3 minutes") } migratedData := logs2.ExtractDataFromLogMessage(match) p("MIGRATED notification received. %v", migratedData) @@ -284,7 +285,7 @@ func TestPushNotifications(t *testing.T) { logger2 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) setupNotificationHooks(client2, tracker2, logger2) commandsRunner2, _ := NewCommandRunner(client2) - t.Log("Second client created") + p("Second client created") // Use a channel to communicate errors from the goroutine errChan := make(chan error, 1) @@ -317,7 +318,7 @@ func TestPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 18) }, 3*time.Minute) if !found { - errChan <- fmt.Errorf("MOVING notification was not received within 2 minutes ON A SECOND CLIENT") + errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A SECOND CLIENT") return } else { p("MOVING notification received on second client %v", logs2.ExtractDataFromLogMessage(match)) @@ -327,7 +328,7 @@ func TestPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ApplyingRelaxedTimeoutDueToPostHandoffMessage) && strings.Contains(s, "30m") }, 3*time.Minute) if !found { - errChan <- fmt.Errorf("relaxed timeout was not applied within 2 minutes ON A SECOND CLIENT") + errChan <- fmt.Errorf("relaxed timeout was not applied within 3 minutes ON A SECOND CLIENT") return } else { p("Relaxed timeout applied on second client") @@ -342,6 +343,61 @@ func TestPushNotifications(t *testing.T) { seqIDToObserve = int64(movingData["seqID"].(float64)) connIDToObserve = uint64(movingData["connID"].(float64)) + // start a third client but don't execute any commands on it + p("Starting a third client to observe notification during moving...") + client3, err := factory.Create("push-notification-client-2", &CreateClientOptions{ + Protocol: 3, // RESP3 required for push notifications + PoolSize: poolSize, + MinIdleConns: minIdleConns, + MaxActiveConns: maxConnections, + MaintNotificationsConfig: &maintnotifications.Config{ + Mode: maintnotifications.ModeEnabled, + HandoffTimeout: 40 * time.Second, // 30 seconds + RelaxedTimeout: 30 * time.Minute, // 30 minutes relaxed timeout for second client + PostHandoffRelaxedDuration: 2 * time.Second, // 2 seconds post-handoff relaxed duration + MaxWorkers: 20, + EndpointType: maintnotifications.EndpointTypeExternalIP, // Use external IP for enterprise + }, + ClientName: "push-notification-test-client-3", + }) + + if err != nil { + ef("failed to create client: %v", err) + } + // setup tracking for second client + tracker3 := NewTrackingNotificationsHook() + logger3 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) + setupNotificationHooks(client3, tracker3, logger3) + commandsRunner3, _ := NewCommandRunner(client3) + p("Third client created") + go commandsRunner3.FireCommandsUntilStop(ctx) + // wait for moving on third client + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { + return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 19) + }, 3*time.Minute) + if !found { + ef("MOVING notification was not received within 3 minutes ON A THIRD CLIENT") + } else { + p("MOVING notification received on third client") + data := logs2.ExtractDataFromLogMessage(match) + mNotif := data["notification"].(string) + // format MOVING endpoint + mNotifParts := strings.Split(mNotif, " ") + if len(mNotifParts) != 4 { + ef("Invalid MOVING notification format: %s", mNotif) + } + mNotifTimeS, err := strconv.Atoi(mNotifParts[2]) + if err != nil { + ef("Invalid timeS in MOVING notification: %s", mNotif) + } + // expect timeS to be less than 15 + if mNotifTimeS < 15 { + ef("Expected timeS < 15, got %d", mNotifTimeS) + } + + p("MOVING notification received on third client. %v", data) + } + commandsRunner3.Stop() // Wait for the goroutine to complete and check for errors if err := <-errChan; err != nil { ef("Second client goroutine error: %v", err) From ebfb7da65cf432096e98c8366ce30490a111e54a Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 16:57:01 +0300 Subject: [PATCH 14/44] dont fail on flaky third client --- maintnotifications/e2e/scenario_push_notifications_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index d839b902f..308d02387 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -376,10 +376,10 @@ func TestPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 19) }, 3*time.Minute) if !found { - ef("MOVING notification was not received within 3 minutes ON A THIRD CLIENT") + p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT") } else { - p("MOVING notification received on third client") data := logs2.ExtractDataFromLogMessage(match) + p("MOVING notification received on third client. %v", data) mNotif := data["notification"].(string) // format MOVING endpoint mNotifParts := strings.Split(mNotif, " ") @@ -394,8 +394,6 @@ func TestPushNotifications(t *testing.T) { if mNotifTimeS < 15 { ef("Expected timeS < 15, got %d", mNotifTimeS) } - - p("MOVING notification received on third client. %v", data) } commandsRunner3.Stop() // Wait for the goroutine to complete and check for errors From ae9f0635a8e38503c8be559248e0929500265855 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Fri, 3 Oct 2025 17:41:03 +0300 Subject: [PATCH 15/44] fi new params --- maintnotifications/e2e/notiftracker_test.go | 31 +++++++++++++++++++ .../e2e/scenario_endpoint_types_test.go | 8 ++--- .../e2e/scenario_push_notifications_test.go | 29 +++++++---------- .../e2e/scenario_stress_test.go | 5 ++- .../e2e/scenario_timeout_configs_test.go | 4 +-- .../e2e/scenario_tls_configs_test.go | 5 ++- 6 files changed, 50 insertions(+), 32 deletions(-) diff --git a/maintnotifications/e2e/notiftracker_test.go b/maintnotifications/e2e/notiftracker_test.go index f2a97286f..b35378dad 100644 --- a/maintnotifications/e2e/notiftracker_test.go +++ b/maintnotifications/e2e/notiftracker_test.go @@ -81,6 +81,37 @@ func (tnh *TrackingNotificationsHook) Clear() { tnh.migratedCount.Store(0) tnh.failingOverCount.Store(0) } +// wait for notification in prehook +func (tnh *TrackingNotificationsHook) FindOrWaitForNotification(notificationType string, timeout time.Duration) (notification []interface{}, found bool) { + if notification, found := tnh.FindNotification(notificationType); found { + return notification, true + } + + // wait for notification + timeoutCh := time.After(timeout) + ticker := time.NewTicker(100 * time.Millisecond) + for { + select { + case <-timeoutCh: + return nil, false + case <-ticker.C: + if notification, found := tnh.FindNotification(notificationType); found { + return notification, true + } + } + } +} + +func (tnh *TrackingNotificationsHook) FindNotification(notificationType string) (notification []interface{}, found bool) { + tnh.mutex.RLock() + defer tnh.mutex.RUnlock() + for _, event := range tnh.diagnosticsLog { + if event.Type == notificationType { + return event.Details["notification"].([]interface{}), true + } + } + return nil, false +} // PreHook captures timeout-related events before processing func (tnh *TrackingNotificationsHook) PreHook(_ context.Context, notificationCtx push.NotificationHandlerContext, notificationType string, notification []interface{}) ([]interface{}, bool) { diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index c4799a2b0..1560a899b 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -173,8 +173,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -223,7 +222,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -266,8 +265,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "bind", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 308d02387..486ff5745 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -147,8 +147,7 @@ func TestPushNotifications(t *testing.T) { failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -199,7 +198,7 @@ func TestPushNotifications(t *testing.T) { migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -251,8 +250,7 @@ func TestPushNotifications(t *testing.T) { bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "bind", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -297,7 +295,7 @@ func TestPushNotifications(t *testing.T) { } }() - p("Waiting for MOVING notification on second client") + p("Waiting for MOVING notification on first client") match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") }, 3*time.Minute) @@ -314,6 +312,7 @@ func TestPushNotifications(t *testing.T) { // wait for moving on second client // we know the maxconn is 15, assuming 16/17 was used to init the second client, so connID 18 should be from the second client // also validate big enough relaxed timeout + p("Waiting for MOVING notification on second client") match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 18) }, 3*time.Minute) @@ -372,27 +371,21 @@ func TestPushNotifications(t *testing.T) { p("Third client created") go commandsRunner3.FireCommandsUntilStop(ctx) // wait for moving on third client - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 19) - }, 3*time.Minute) + movingNotification, found := tracker.FindOrWaitForNotification("MOVING", 3*time.Minute) if !found { p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT") } else { - data := logs2.ExtractDataFromLogMessage(match) p("MOVING notification received on third client. %v", data) - mNotif := data["notification"].(string) - // format MOVING endpoint - mNotifParts := strings.Split(mNotif, " ") - if len(mNotifParts) != 4 { - ef("Invalid MOVING notification format: %s", mNotif) + if len(movingNotification) != 4 { + p("[NOTICE] Invalid MOVING notification format: %s", mNotif) } - mNotifTimeS, err := strconv.Atoi(mNotifParts[2]) + mNotifTimeS, err := strconv.Atoi(movingNotification[2].(string)) if err != nil { - ef("Invalid timeS in MOVING notification: %s", mNotif) + p("[NOTICE] Invalid timeS in MOVING notification: %s", movingNotification) } // expect timeS to be less than 15 if mNotifTimeS < 15 { - ef("Expected timeS < 15, got %d", mNotifTimeS) + p("[NOTICE] Expected timeS < 15, got %d", mNotifTimeS) } } commandsRunner3.Stop() diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index b2358e00e..91c64a757 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -188,15 +188,14 @@ func TestStressPushNotifications(t *testing.T) { resp, err = faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) case "migrate": resp, err = faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, }, }) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index f3f522ea0..37c1a4990 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -195,7 +195,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "cluster_index": "0", "bdb_id": endpointConfig.BdbID, }, }) @@ -244,7 +243,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -275,7 +274,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "bind", Parameters: map[string]interface{}{ - "cluster_index": "0", "bdb_id": endpointConfig.BdbID, }, }) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index b1dc79181..63d12107b 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -197,8 +197,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "cluster_index": "0", - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -242,7 +241,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "cluster_index": "0", + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { From 0ee3a8be05c9c6116513662c98a72b83e5c5c4d8 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 15:38:31 +0300 Subject: [PATCH 16/44] fix test build --- maintnotifications/e2e/scenario_push_notifications_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 486ff5745..96a86ed35 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -375,9 +375,9 @@ func TestPushNotifications(t *testing.T) { if !found { p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT") } else { - p("MOVING notification received on third client. %v", data) + p("MOVING notification received on third client. %v", movingNotification) if len(movingNotification) != 4 { - p("[NOTICE] Invalid MOVING notification format: %s", mNotif) + p("[NOTICE] Invalid MOVING notification format: %s", movingNotification) } mNotifTimeS, err := strconv.Atoi(movingNotification[2].(string)) if err != nil { From 61998d5b8dbb2301d28a10d97d958963afdfc027 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 16:52:57 +0300 Subject: [PATCH 17/44] more time for migrating --- .../e2e/scenario_endpoint_types_test.go | 2 +- .../e2e/scenario_push_notifications_test.go | 4 +- .../e2e/scenario_timeout_configs_test.go | 8 ++-- .../e2e/scenario_tls_configs_test.go | 46 +------------------ 4 files changed, 8 insertions(+), 52 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 1560a899b..c73a01cfe 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -232,7 +232,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") - }, 30*time.Second) + }, 60*time.Second) if !found { ef("MIGRATING notification was not received for %s endpoint type", endpointTest.name) } diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 96a86ed35..743115127 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -207,12 +207,12 @@ func TestPushNotifications(t *testing.T) { go func() { match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") - }, 20*time.Second) + }, 60*time.Second) commandsRunner.Stop() }() commandsRunner.FireCommandsUntilStop(ctx) if !found { - ef("MIGRATING notification for migrate action was not received within 20 seconds") + ef("MIGRATING notification for migrate action was not received within 60 seconds") } migrateData := logs2.ExtractDataFromLogMessage(match) seqIDToObserve = int64(migrateData["seqID"].(float64)) diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 37c1a4990..8f2a1d974 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -195,7 +195,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -243,7 +243,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { @@ -253,7 +253,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") - }, 30*time.Second) + }, 60*time.Second) if !found { ef("MIGRATING notification was not received for %s timeout config", timeoutTest.name) } @@ -274,7 +274,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "bind", Parameters: map[string]interface{}{ - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) if err != nil { diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 63d12107b..8f2734081 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -192,50 +192,6 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { commandsRunner.FireCommandsUntilStop(ctx) }() - // Test failover with this TLS configuration - p("Testing failover with %s TLS configuration...", tlsTest.name) - failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ - Type: "failover", - Parameters: map[string]interface{}{ - "bdb_id": endpointConfig.BdbID, - }, - }) - if err != nil { - ef("Failed to trigger failover action for %s: %v", tlsTest.name, err) - } - - // Wait for FAILING_OVER notification - match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") - }, 3*time.Minute) - if !found { - ef("FAILING_OVER notification was not received for %s TLS config", tlsTest.name) - } - failingOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILING_OVER notification received for %s. %v", tlsTest.name, failingOverData) - - // Wait for FAILED_OVER notification - seqIDToObserve := int64(failingOverData["seqID"].(float64)) - connIDToObserve := uint64(failingOverData["connID"].(float64)) - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1) - }, 3*time.Minute) - if !found { - ef("FAILED_OVER notification was not received for %s TLS config", tlsTest.name) - } - failedOverData := logs2.ExtractDataFromLogMessage(match) - p("FAILED_OVER notification received for %s. %v", tlsTest.name, failedOverData) - - // Wait for failover to complete - status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(240*time.Second), - WithPollInterval(2*time.Second), - ) - if err != nil { - ef("[FI] Failover action failed for %s: %v", tlsTest.name, err) - } - p("[FI] Failover action completed for %s: %s", tlsTest.name, status.Status) - // Test migration with this TLS configuration p("Testing migration with %s TLS configuration...", tlsTest.name) migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ @@ -251,7 +207,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") - }, 30*time.Second) + }, 60*time.Second) if !found { ef("MIGRATING notification was not received for %s TLS config", tlsTest.name) } From 0c8434cd51fb2d0273c4cbbb677e91013b2a1c7e Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 16:56:03 +0300 Subject: [PATCH 18/44] first wait for FI action, then assert notification --- .../e2e/scenario_timeout_configs_test.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 8f2a1d974..ae169a68c 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -250,6 +250,17 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ef("Failed to trigger migrate action for %s: %v", timeoutTest.name, err) } + // Wait for migration to complete + status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), + ) + if err != nil { + ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) + } + + p("[FI] Migrate action completed for %s: %s", timeoutTest.name, status.Status) + // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") @@ -260,15 +271,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { migrateData := logs2.ExtractDataFromLogMessage(match) p("MIGRATING notification received for %s: %v", timeoutTest.name, migrateData) - // Wait for migration to complete - status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, - WithMaxWaitTime(240*time.Second), - WithPollInterval(2*time.Second), - ) - if err != nil { - ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) - } - p("[FI] Migrate action completed for %s: %s", timeoutTest.name, status.Status) // do a bind action bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ From d8d55f9a216da62287babaad5e3f0e0f2327d588 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 17:12:00 +0300 Subject: [PATCH 19/44] fix test build --- maintnotifications/e2e/scenario_tls_configs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 8f2734081..7d24a71e9 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -205,7 +205,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { } // Wait for MIGRATING notification - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found := logCollector.WaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 60*time.Second) if !found { @@ -215,7 +215,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { p("MIGRATING notification received for %s: %v", tlsTest.name, migrateData) // Wait for migration to complete - status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, + status, err := faultInjector.WaitForAction(ctx, migrateResp.ActionID, WithMaxWaitTime(240*time.Second), WithPollInterval(2*time.Second), ) From 93ef6cc032cf9754bddb110b19d5a8b2b21f994d Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 17:37:56 +0300 Subject: [PATCH 20/44] fix tests --- .../e2e/scenario_endpoint_types_test.go | 25 +++++++++++-------- .../e2e/scenario_push_notifications_test.go | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index c73a01cfe..fff93e444 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -86,7 +86,10 @@ func TestEndpointTypesPushNotifications(t *testing.T) { t.Run(endpointTest.name, func(t *testing.T) { // Clear logs between endpoint type tests logCollector.Clear() - dump = true // reset dump flag + // reset errors detected flag + errorsDetected = false + // reset dump flag + dump = true // redefine p and e for each test to get // proper test name in logs and proper test failures var p = func(format string, args ...interface{}) { @@ -229,16 +232,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { ef("Failed to trigger migrate action for %s: %v", endpointTest.name, err) } - // Wait for MIGRATING notification - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") - }, 60*time.Second) - if !found { - ef("MIGRATING notification was not received for %s endpoint type", endpointTest.name) - } - migrateData := logs2.ExtractDataFromLogMessage(match) - p("MIGRATING notification received for %s: %v", endpointTest.name, migrateData) - // Wait for migration to complete status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, WithMaxWaitTime(240*time.Second), @@ -249,6 +242,16 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } p("[FI] Migrate action completed for %s: %s", endpointTest.name, status.Status) + // Wait for MIGRATING notification + match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") + }, 60*time.Second) + if !found { + ef("MIGRATING notification was not received for %s endpoint type", endpointTest.name) + } + migrateData := logs2.ExtractDataFromLogMessage(match) + p("MIGRATING notification received for %s: %v", endpointTest.name, migrateData) + // Wait for MIGRATED notification seqIDToObserve = int64(migrateData["seqID"].(float64)) connIDToObserve = uint64(migrateData["connID"].(float64)) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 743115127..a0391a08a 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -379,7 +379,7 @@ func TestPushNotifications(t *testing.T) { if len(movingNotification) != 4 { p("[NOTICE] Invalid MOVING notification format: %s", movingNotification) } - mNotifTimeS, err := strconv.Atoi(movingNotification[2].(string)) + mNotifTimeS, err := movingNotification[2].(int64) if err != nil { p("[NOTICE] Invalid timeS in MOVING notification: %s", movingNotification) } From 3699acf7042291082c733357cb12e42e68617b5c Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 18:10:27 +0300 Subject: [PATCH 21/44] fix tests --- maintnotifications/e2e/scenario_push_notifications_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index a0391a08a..ae3e020ae 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "strconv" "strings" "testing" "time" @@ -379,8 +378,8 @@ func TestPushNotifications(t *testing.T) { if len(movingNotification) != 4 { p("[NOTICE] Invalid MOVING notification format: %s", movingNotification) } - mNotifTimeS, err := movingNotification[2].(int64) - if err != nil { + mNotifTimeS, ok := movingNotification[2].(int64) + if !ok { p("[NOTICE] Invalid timeS in MOVING notification: %s", movingNotification) } // expect timeS to be less than 15 From 012d566766c5e74864e5470bea1cd50bdde0ef5e Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 18:57:54 +0300 Subject: [PATCH 22/44] change output --- .../e2e/scenario_endpoint_types_test.go | 26 +++++---- .../e2e/scenario_push_notifications_test.go | 38 +++++-------- .../e2e/scenario_stress_test.go | 40 +++++++------- .../e2e/scenario_timeout_configs_test.go | 54 +++++++++---------- .../e2e/scenario_tls_configs_test.go | 53 +++++++++--------- 5 files changed, 95 insertions(+), 116 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index fff93e444..a41d888df 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "os" + "runtime" "strings" "testing" "time" @@ -93,18 +94,20 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // redefine p and e for each test to get // proper test name in logs and proper test failures var p = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][ENDPOINT-TYPES] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - format = "[%s][ENDPOINT-TYPES][ERROR] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][ENDPOINT-TYPES][ERROR] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Errorf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } var ef = func(format string, args ...interface{}) { @@ -146,10 +149,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { logger := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) setupNotificationHooks(client, tracker, logger) defer func() { - if dump { - p("Tracker analysis for %s:", endpointTest.name) - tracker.GetAnalysis().Print(t) - } tracker.Clear() }() @@ -218,7 +217,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) } - p("[FI] Failover action completed for %s: %s", endpointTest.name, status.Status) + p("[FI] Failover action completed for %s: %v", endpointTest.name, status) // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) @@ -240,7 +239,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) } - p("[FI] Migrate action completed for %s: %s", endpointTest.name, status.Status) + p("[FI] Migrate action completed for %s: %v", endpointTest.name, status) // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { @@ -334,7 +333,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("Bind action failed for %s: %v", endpointTest.name, err) } - p("Bind action completed for %s: %s", endpointTest.name, bindStatus.Status) + p("Bind action completed for %s: %v", endpointTest.name, bindStatus) // Continue traffic for analysis time.Sleep(30 * time.Second) @@ -382,7 +381,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { tracker.Clear() ef("[FAIL] Errors detected with %s endpoint type", endpointTest.name) } - dump = false p("Endpoint type %s test completed successfully", endpointTest.name) logCollector.GetAnalysis().Print(t) trackerAnalysis.Print(t) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index ae3e020ae..5f2eaca83 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "strings" "testing" "time" @@ -30,21 +31,23 @@ func TestPushNotifications(t *testing.T) { var found bool var status *ActionStatusResponse + var errorsDetected = false var p = func(format string, args ...interface{}) { - format = "[%s][PUSH-NOTIFICATIONS] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][PUSH-NOTIFICATIONS] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } - var errorsDetected = false var e = func(format string, args ...interface{}) { errorsDetected = true - format = "[%s][PUSH-NOTIFICATIONS][ERROR] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][PUSH-NOTIFICATIONS][ERROR] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Errorf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } var ef = func(format string, args ...interface{}) { @@ -57,12 +60,6 @@ func TestPushNotifications(t *testing.T) { logCollector.ClearLogs() defer func() { - if dump { - p("Dumping logs...") - logCollector.DumpLogs() - p("Log Analysis:") - logCollector.GetAnalysis().Print(t) - } logCollector.Clear() }() @@ -103,10 +100,6 @@ func TestPushNotifications(t *testing.T) { } defer func() { - if dump { - p("Pool stats:") - factory.PrintPoolStats(t) - } factory.DestroyAll() }() @@ -115,9 +108,6 @@ func TestPushNotifications(t *testing.T) { logger := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) setupNotificationHooks(client, tracker, logger) defer func() { - if dump { - tracker.GetAnalysis().Print(t) - } tracker.Clear() }() @@ -188,7 +178,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed: %v", err) } - fmt.Printf("[FI] Failover action completed: %s\n", status.Status) + p("[FI] Failover action completed: %v", status) p("FAILING_OVER / FAILED_OVER notifications test completed successfully") @@ -225,7 +215,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed: %v", err) } - fmt.Printf("[FI] Migrate action completed: %s\n", status.Status) + p("[FI] Migrate action completed: %v", status) go func() { p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) @@ -401,7 +391,7 @@ func TestPushNotifications(t *testing.T) { ef("Bind action failed: %v", err) } - p("Bind action completed: %s", bindStatus.Status) + p("Bind action completed: %v", bindStatus) p("MOVING notification test completed successfully") @@ -515,8 +505,6 @@ func TestPushNotifications(t *testing.T) { } p("Analysis complete, no errors found") - // print analysis here, don't dump logs later - dump = false allLogsAnalysis.Print(t) trackerAnalysis.Print(t) p("Command runner stats:") diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 91c64a757..2d89d01de 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "sync" "testing" "time" @@ -23,21 +24,25 @@ func TestStressPushNotifications(t *testing.T) { defer cancel() var dump = true + var errorsDetected = false + var p = func(format string, args ...interface{}) { - format = "[%s][STRESS] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][STRESS] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } - var errorsDetected = false var e = func(format string, args ...interface{}) { errorsDetected = true - format = "[%s][STRESS][ERROR] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][STRESS][ERROR] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Errorf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } + var ef = func(format string, args ...interface{}) { errorsDetected = true format = "[%s][STRESS][ERROR] " + format @@ -48,12 +53,6 @@ func TestStressPushNotifications(t *testing.T) { logCollector.ClearLogs() defer func() { - if dump { - p("Dumping logs...") - logCollector.DumpLogs() - p("Log Analysis:") - logCollector.GetAnalysis().Print(t) - } logCollector.Clear() }() @@ -118,10 +117,6 @@ func TestStressPushNotifications(t *testing.T) { if dump { p("Pool stats:") factory.PrintPoolStats(t) - for i, tracker := range trackers { - p("Stress client %d analysis:", i) - tracker.GetAnalysis().Print(t) - } } for _, runner := range commandRunners { runner.Stop() @@ -195,7 +190,7 @@ func TestStressPushNotifications(t *testing.T) { resp, err = faultInjector.TriggerAction(ctx, ActionRequest{ Type: "migrate", Parameters: map[string]interface{}{ - "bdb_id": endpointConfig.BdbID, + "bdb_id": endpointConfig.BdbID, }, }) } @@ -297,6 +292,15 @@ func TestStressPushNotifications(t *testing.T) { if errorsDetected { ef("Errors detected under stress") + logCollector.DumpLogs() + for i, tracker := range trackers { + p("=== Stress Client %d Analysis ===", i) + tracker.GetAnalysis().Print(t) + } + logCollector.Clear() + for _, tracker := range trackers { + tracker.Clear() + } } dump = false diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index ae169a68c..3a3d59eac 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "strings" "testing" "time" @@ -23,11 +24,23 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { defer cancel() var dump = true + + var errorsDetected = false var p = func(format string, args ...interface{}) { - format = "[%s][TIMEOUT-CONFIGS] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][TIMEOUT-CONFIGS] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) + } + + var e = func(format string, args ...interface{}) { + errorsDetected = true + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][TIMEOUT-CONFIGS][ERROR] " + format + "\n" + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } // Test different timeout configurations @@ -67,12 +80,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { logCollector.ClearLogs() defer func() { - if dump { - p("Dumping logs...") - logCollector.DumpLogs() - p("Log Analysis:") - logCollector.GetAnalysis().Print(t) - } logCollector.Clear() }() @@ -100,22 +107,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { // Test each timeout configuration for _, timeoutTest := range timeoutConfigs { t.Run(timeoutTest.name, func(t *testing.T) { - // redefine p and e for each test to get - // proper test name in logs and proper test failures - var p = func(format string, args ...interface{}) { - format = "[%s][TIMEOUT-CONFIGS] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) - } - - var e = func(format string, args ...interface{}) { - format = "[%s][TIMEOUT-CONFIGS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Errorf(format, args...) - } - + errorsDetected = false var ef = func(format string, args ...interface{}) { format = "[%s][TIMEOUT-CONFIGS][ERROR] " + format ts := time.Now().Format("15:04:05.000") @@ -157,10 +149,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { logger := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) setupNotificationHooks(client, tracker, logger) defer func() { - if dump { - p("Tracker analysis for %s:", timeoutTest.name) - tracker.GetAnalysis().Print(t) - } tracker.Clear() }() @@ -271,7 +259,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { migrateData := logs2.ExtractDataFromLogMessage(match) p("MIGRATING notification received for %s: %v", timeoutTest.name, migrateData) - // do a bind action bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "bind", @@ -358,6 +345,13 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { e("Expected successful handoffs with %s config, got none", timeoutTest.name) } + if errorsDetected { + logCollector.DumpLogs() + trackerAnalysis.Print(t) + logCollector.Clear() + tracker.Clear() + ef("[FAIL] Errors detected with %s timeout config", timeoutTest.name) + } p("Timeout configuration %s test completed successfully in %v", timeoutTest.name, testDuration) p("Command runner stats:") p("Operations: %d, Errors: %d, Timeout Errors: %d", diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 7d24a71e9..e783214a9 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "strings" "testing" "time" @@ -24,11 +25,22 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { defer cancel() var dump = true + var errorsDetected = false var p = func(format string, args ...interface{}) { - format = "[%s][TLS-CONFIGS] " + format + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][TLS-CONFIGS] " + format + "\n" ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) + } + + var e = func(format string, args ...interface{}) { + errorsDetected = true + _, filename, line, _ := runtime.Caller(1) + format = "%s:%d [%s][TLS-CONFIGS][ERROR] " + format + "\n" + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{filename, line, ts}, args...) + fmt.Printf(format, args...) } // Test different TLS configurations @@ -64,12 +76,6 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { logCollector.ClearLogs() defer func() { - if dump { - p("Dumping logs...") - logCollector.DumpLogs() - p("Log Analysis:") - logCollector.GetAnalysis().Print(t) - } logCollector.Clear() }() @@ -97,28 +103,14 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { // Test each TLS configuration for _, tlsTest := range tlsConfigs { t.Run(tlsTest.name, func(t *testing.T) { - // redefine p and e for each test to get - // proper test name in logs and proper test failures - var p = func(format string, args ...interface{}) { - format = "[%s][TLS-CONFIGS] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) - } - - var e = func(format string, args ...interface{}) { - format = "[%s][TLS-CONFIGS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Errorf(format, args...) - } - + errorsDetected = false var ef = func(format string, args ...interface{}) { format = "[%s][TLS-CONFIGS][ERROR] " + format ts := time.Now().Format("15:04:05.000") args = append([]interface{}{ts}, args...) t.Fatalf(format, args...) } + if tlsTest.skipReason != "" { t.Skipf("Skipping %s: %s", tlsTest.name, tlsTest.skipReason) } @@ -159,10 +151,6 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { logger := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug)) setupNotificationHooks(client, tracker, logger) defer func() { - if dump { - p("Tracker analysis for %s:", tlsTest.name) - tracker.GetAnalysis().Print(t) - } tracker.Clear() }() @@ -249,6 +237,13 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { e("Expected MIGRATING notifications with %s TLS config, got none", tlsTest.name) } + if errorsDetected { + logCollector.DumpLogs() + trackerAnalysis.Print(t) + logCollector.Clear() + tracker.Clear() + ef("[FAIL] Errors detected with %s TLS config", tlsTest.name) + } // TLS-specific validations stats := commandsRunner.GetStats() switch tlsTest.name { From 560f0b7fa1fb68a10ae1d85686ef0067f4c6b345 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 19:21:22 +0300 Subject: [PATCH 23/44] global print logs for tests --- .../e2e/scenario_endpoint_types_test.go | 20 ++++--------------- .../e2e/scenario_push_notifications_test.go | 20 ++++--------------- .../e2e/scenario_stress_test.go | 20 ++++--------------- .../e2e/scenario_timeout_configs_test.go | 19 ++++-------------- .../e2e/scenario_tls_configs_test.go | 19 ++++-------------- maintnotifications/e2e/utils_test.go | 19 ++++++++++++++++++ 6 files changed, 39 insertions(+), 78 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index a41d888df..c927ac96b 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "os" - "runtime" "strings" "testing" "time" @@ -94,28 +93,17 @@ func TestEndpointTypesPushNotifications(t *testing.T) { // redefine p and e for each test to get // proper test name in logs and proper test failures var p = func(format string, args ...interface{}) { - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][ENDPOINT-TYPES] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("ENDPOINT-TYPES", false, format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][ENDPOINT-TYPES][ERROR] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("ENDPOINT-TYPES", true, format, args...) } var ef = func(format string, args ...interface{}) { - errorsDetected = true - format = "[%s][ENDPOINT-TYPES][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Fatalf(format, args...) + printLog("ENDPOINT-TYPES", true, format, args...) + t.FailNow() } p("Testing endpoint type: %s - %s", endpointTest.name, endpointTest.description) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 5f2eaca83..cc6d0d019 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "runtime" "strings" "testing" "time" @@ -34,28 +33,17 @@ func TestPushNotifications(t *testing.T) { var errorsDetected = false var p = func(format string, args ...interface{}) { - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][PUSH-NOTIFICATIONS] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("PUSH-NOTIFICATIONS", false, format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][PUSH-NOTIFICATIONS][ERROR] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("PUSH-NOTIFICATIONS", true, format, args...) } var ef = func(format string, args ...interface{}) { - errorsDetected = true - format = "[%s][PUSH-NOTIFICATIONS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Fatalf(format, args...) + printLog("PUSH-NOTIFICATIONS", true, format, args...) + t.FailNow() } logCollector.ClearLogs() diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 2d89d01de..90856d6ab 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "runtime" "sync" "testing" "time" @@ -27,28 +26,17 @@ func TestStressPushNotifications(t *testing.T) { var errorsDetected = false var p = func(format string, args ...interface{}) { - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][STRESS] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("STRESS", false, format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][STRESS][ERROR] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("STRESS", true, format, args...) } var ef = func(format string, args ...interface{}) { - errorsDetected = true - format = "[%s][STRESS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Fatalf(format, args...) + printLog("STRESS", true, format, args...) + t.FailNow() } logCollector.ClearLogs() diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 3a3d59eac..eadace481 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "runtime" "strings" "testing" "time" @@ -27,20 +26,12 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { var errorsDetected = false var p = func(format string, args ...interface{}) { - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][TIMEOUT-CONFIGS] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("TIMEOUT-CONFIGS", false, format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][TIMEOUT-CONFIGS][ERROR] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("TIMEOUT-CONFIGS", true, format, args...) } // Test different timeout configurations @@ -109,10 +100,8 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { t.Run(timeoutTest.name, func(t *testing.T) { errorsDetected = false var ef = func(format string, args ...interface{}) { - format = "[%s][TIMEOUT-CONFIGS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Fatalf(format, args...) + printLog("TIMEOUT-CONFIGS", true, format, args...) + t.FailNow() } p("Testing timeout configuration: %s - %s", timeoutTest.name, timeoutTest.description) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index e783214a9..59b51445c 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "runtime" "strings" "testing" "time" @@ -27,20 +26,12 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { var dump = true var errorsDetected = false var p = func(format string, args ...interface{}) { - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][TLS-CONFIGS] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("TLS-CONFIGS", false, format, args...) } var e = func(format string, args ...interface{}) { errorsDetected = true - _, filename, line, _ := runtime.Caller(1) - format = "%s:%d [%s][TLS-CONFIGS][ERROR] " + format + "\n" - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{filename, line, ts}, args...) - fmt.Printf(format, args...) + printLog("TLS-CONFIGS", true, format, args...) } // Test different TLS configurations @@ -105,10 +96,8 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { t.Run(tlsTest.name, func(t *testing.T) { errorsDetected = false var ef = func(format string, args ...interface{}) { - format = "[%s][TLS-CONFIGS][ERROR] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Fatalf(format, args...) + printLog("TLS-CONFIGS", true, format, args...) + t.FailNow() } if tlsTest.skipReason != "" { diff --git a/maintnotifications/e2e/utils_test.go b/maintnotifications/e2e/utils_test.go index eb3cbe0b0..1773200ff 100644 --- a/maintnotifications/e2e/utils_test.go +++ b/maintnotifications/e2e/utils_test.go @@ -1,5 +1,12 @@ package e2e +import ( + "fmt" + "path/filepath" + "runtime" + "time" +) + func isTimeout(errMsg string) bool { return contains(errMsg, "i/o timeout") || contains(errMsg, "deadline exceeded") || @@ -42,3 +49,15 @@ func min(a, b int) int { } return b } + +func printLog(group string, isError bool, format string, args ...interface{}) { + _, filename, line, _ := runtime.Caller(2) + filename = filepath.Base(filename) + if isError { + format = "%s:%d [%s][%s][ERROR] " + format + "\n" + } + format = "%s:%d [%s][%s] " + format + "\n" + ts := time.Now().Format("15:04:05.000") + args = append([]interface{}{filename, line, ts, group}, args...) + fmt.Printf(format, args...) +} From 761daafcc7ebbb3693720f46f8270e34735cf399 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 19:26:27 +0300 Subject: [PATCH 24/44] better output --- maintnotifications/e2e/scenario_endpoint_types_test.go | 6 +++--- maintnotifications/e2e/scenario_push_notifications_test.go | 6 +++--- maintnotifications/e2e/scenario_stress_test.go | 4 ++-- maintnotifications/e2e/scenario_timeout_configs_test.go | 6 +++--- maintnotifications/e2e/scenario_tls_configs_test.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index c927ac96b..2facee010 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -205,7 +205,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) } - p("[FI] Failover action completed for %s: %v", endpointTest.name, status) + p("[FI] Failover action completed for %s: %+v", endpointTest.name, status) // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) @@ -227,7 +227,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) } - p("[FI] Migrate action completed for %s: %v", endpointTest.name, status) + p("[FI] Migrate action completed for %s: %+v", endpointTest.name, status) // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { @@ -321,7 +321,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("Bind action failed for %s: %v", endpointTest.name, err) } - p("Bind action completed for %s: %v", endpointTest.name, bindStatus) + p("Bind action completed for %s: %+v", endpointTest.name, bindStatus) // Continue traffic for analysis time.Sleep(30 * time.Second) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index cc6d0d019..23e8db391 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -166,7 +166,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed: %v", err) } - p("[FI] Failover action completed: %v", status) + p("[FI] Failover action completed: %+v", status) p("FAILING_OVER / FAILED_OVER notifications test completed successfully") @@ -203,7 +203,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed: %v", err) } - p("[FI] Migrate action completed: %v", status) + p("[FI] Migrate action completed: %+v", status) go func() { p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) @@ -379,7 +379,7 @@ func TestPushNotifications(t *testing.T) { ef("Bind action failed: %v", err) } - p("Bind action completed: %v", bindStatus) + p("Bind action completed: %+v", bindStatus) p("MOVING notification test completed successfully") diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 90856d6ab..edb8ed8b4 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -199,10 +199,10 @@ func TestStressPushNotifications(t *testing.T) { } actionMutex.Lock() - actionResults = append(actionResults, fmt.Sprintf("%s: %s", actionName, status.Status)) + actionResults = append(actionResults, fmt.Sprintf("%s: %+v", actionName, status)) actionMutex.Unlock() - p("[FI] %s action completed: %s", actionName, status.Status) + p("[FI] %s action completed: %+v", actionName, status) }(action.name, action.action, action.delay) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index eadace481..2509f2fde 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -209,7 +209,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Failover action completed for %s: %s", timeoutTest.name, status.Status) + p("[FI] Failover action completed for %s: %+v", timeoutTest.name, status) // Continue traffic to observe timeout behavior p("Continuing traffic for %v to observe timeout behavior...", timeoutTest.relaxedTimeout*2) @@ -236,7 +236,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Migrate action completed for %s: %s", timeoutTest.name, status.Status) + p("[FI] Migrate action completed for %s: %+v", timeoutTest.name, status) // Wait for MIGRATING notification match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { @@ -265,7 +265,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Bind action completed for %s: %s", timeoutTest.name, status.Status) + p("[FI] Bind action completed for %s: %+v", timeoutTest.name, status) // waiting for moving notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 59b51445c..a6a3e92c9 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -199,7 +199,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", tlsTest.name, err) } - p("[FI] Migrate action completed for %s: %s", tlsTest.name, status.Status) + p("[FI] Migrate action completed for %s: %+v", tlsTest.name, status) // Continue traffic for a bit to observe TLS behavior time.Sleep(5 * time.Second) From bbd7b8dc21e3ba1738bfee77ce5c4266eebea78a Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 19:45:29 +0300 Subject: [PATCH 25/44] fix error format --- maintnotifications/e2e/utils_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maintnotifications/e2e/utils_test.go b/maintnotifications/e2e/utils_test.go index 1773200ff..66be7b0d4 100644 --- a/maintnotifications/e2e/utils_test.go +++ b/maintnotifications/e2e/utils_test.go @@ -53,11 +53,11 @@ func min(a, b int) int { func printLog(group string, isError bool, format string, args ...interface{}) { _, filename, line, _ := runtime.Caller(2) filename = filepath.Base(filename) + finalFormat := "%s:%d [%s][%s] " + format + "\n" if isError { - format = "%s:%d [%s][%s][ERROR] " + format + "\n" + finalFormat = "%s:%d [%s][%s][ERROR] " + format + "\n" } - format = "%s:%d [%s][%s] " + format + "\n" ts := time.Now().Format("15:04:05.000") args = append([]interface{}{filename, line, ts, group}, args...) - fmt.Printf(format, args...) + fmt.Printf(finalFormat, args...) } From e27d232dca7c1fea5d8a567e9ce3155d6253ad56 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 20:09:24 +0300 Subject: [PATCH 26/44] maybe the notification is already received --- .../e2e/scenario_endpoint_types_test.go | 14 ++++++++------ .../e2e/scenario_push_notifications_test.go | 2 +- .../e2e/scenario_timeout_configs_test.go | 2 +- .../e2e/scenario_tls_configs_test.go | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 2facee010..a52609de7 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -60,6 +60,13 @@ func TestEndpointTypesPushNotifications(t *testing.T) { logCollector.Clear() }() + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) + } + + // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { @@ -67,11 +74,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } endpointConfig := factory.GetConfig() - // Create fault injector - faultInjector, err := CreateTestFaultInjector() - if err != nil { - t.Fatalf("[ERROR] Failed to create fault injector: %v", err) - } defer func() { if dump { @@ -230,7 +232,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { p("[FI] Migrate action completed for %s: %+v", endpointTest.name, status) // Wait for MIGRATING notification - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 60*time.Second) if !found { diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 23e8db391..a90a46cdb 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -182,7 +182,7 @@ func TestPushNotifications(t *testing.T) { ef("Failed to trigger migrate action: %v", err) } go func() { - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 60*time.Second) commandsRunner.Stop() diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 2509f2fde..bd808c592 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -239,7 +239,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { p("[FI] Migrate action completed for %s: %+v", timeoutTest.name, status) // Wait for MIGRATING notification - match, found = logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 60*time.Second) if !found { diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index a6a3e92c9..bcb709aea 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -182,7 +182,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { } // Wait for MIGRATING notification - match, found := logCollector.WaitForLogMatchFunc(func(s string) bool { + match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING") }, 60*time.Second) if !found { From 7329e31e2a6f7d6043558ace35e38a852a39a28f Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 20:40:14 +0300 Subject: [PATCH 27/44] second and third client fix --- .../e2e/scenario_endpoint_types_test.go | 6 +++--- .../e2e/scenario_push_notifications_test.go | 19 +++++++++++-------- .../e2e/scenario_stress_test.go | 4 ++-- .../e2e/scenario_timeout_configs_test.go | 7 ++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index a52609de7..90e5c1378 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -207,7 +207,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) } - p("[FI] Failover action completed for %s: %+v", endpointTest.name, status) + p("[FI] Failover action completed for %s: %+v", endpointTest.name, status.Status) // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) @@ -229,7 +229,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) } - p("[FI] Migrate action completed for %s: %+v", endpointTest.name, status) + p("[FI] Migrate action completed for %s: %+v", endpointTest.name, status.Status) // Wait for MIGRATING notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { @@ -323,7 +323,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("Bind action failed for %s: %v", endpointTest.name, err) } - p("Bind action completed for %s: %+v", endpointTest.name, bindStatus) + p("Bind action completed for %s: %+v", endpointTest.name, bindStatus.Status) // Continue traffic for analysis time.Sleep(30 * time.Second) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index a90a46cdb..8f327185a 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -166,7 +166,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed: %v", err) } - p("[FI] Failover action completed: %+v", status) + p("[FI] Failover action completed: %+v", status.Status) p("FAILING_OVER / FAILED_OVER notifications test completed successfully") @@ -203,7 +203,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed: %v", err) } - p("[FI] Migrate action completed: %+v", status) + p("[FI] Migrate action completed: %+v", status.Status) go func() { p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) @@ -266,6 +266,9 @@ func TestPushNotifications(t *testing.T) { errChan := make(chan error, 1) go func() { + var match string + var matchNotif []interface{} + var found bool defer func() { if r := recover(); r != nil { errChan <- fmt.Errorf("goroutine panic: %v", r) @@ -290,15 +293,14 @@ func TestPushNotifications(t *testing.T) { // we know the maxconn is 15, assuming 16/17 was used to init the second client, so connID 18 should be from the second client // also validate big enough relaxed timeout p("Waiting for MOVING notification on second client") - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") && connID(s, 18) - }, 3*time.Minute) + matchNotif, found = tracker2.FindOrWaitForNotification("MOVING", 3*time.Minute) if !found { errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A SECOND CLIENT") return } else { - p("MOVING notification received on second client %v", logs2.ExtractDataFromLogMessage(match)) + p("MOVING notification received on second client %v", matchNotif) } + // wait for relaxation of 30m match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ApplyingRelaxedTimeoutDueToPostHandoffMessage) && strings.Contains(s, "30m") @@ -319,6 +321,7 @@ func TestPushNotifications(t *testing.T) { seqIDToObserve = int64(movingData["seqID"].(float64)) connIDToObserve = uint64(movingData["connID"].(float64)) + time.Sleep(3 * time.Second) // start a third client but don't execute any commands on it p("Starting a third client to observe notification during moving...") client3, err := factory.Create("push-notification-client-2", &CreateClientOptions{ @@ -348,7 +351,7 @@ func TestPushNotifications(t *testing.T) { p("Third client created") go commandsRunner3.FireCommandsUntilStop(ctx) // wait for moving on third client - movingNotification, found := tracker.FindOrWaitForNotification("MOVING", 3*time.Minute) + movingNotification, found := tracker3.FindOrWaitForNotification("MOVING", 3*time.Minute) if !found { p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT") } else { @@ -379,7 +382,7 @@ func TestPushNotifications(t *testing.T) { ef("Bind action failed: %v", err) } - p("Bind action completed: %+v", bindStatus) + p("Bind action completed: %+v", bindStatus.Status) p("MOVING notification test completed successfully") diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index edb8ed8b4..ed14f0c67 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -199,10 +199,10 @@ func TestStressPushNotifications(t *testing.T) { } actionMutex.Lock() - actionResults = append(actionResults, fmt.Sprintf("%s: %+v", actionName, status)) + actionResults = append(actionResults, fmt.Sprintf("%s: %+v", actionName, status.Status)) actionMutex.Unlock() - p("[FI] %s action completed: %+v", actionName, status) + p("[FI] %s action completed: %+v", actionName, status.Status) }(action.name, action.action, action.delay) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index bd808c592..9b6e346a3 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -209,7 +209,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Failover action completed for %s: %+v", timeoutTest.name, status) + p("[FI] Failover action completed for %s: %+v", timeoutTest.name, status.Status) // Continue traffic to observe timeout behavior p("Continuing traffic for %v to observe timeout behavior...", timeoutTest.relaxedTimeout*2) @@ -236,7 +236,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Migrate action completed for %s: %+v", timeoutTest.name, status) + p("[FI] Migrate action completed for %s: %+v", timeoutTest.name, status.Status) // Wait for MIGRATING notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { @@ -265,7 +265,8 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Bind action completed for %s: %+v", timeoutTest.name, status) + p("[FI] Bind action completed for %s: %+v", timeoutTest.name, status.Status) + // waiting for moving notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") From 1adb5769288d1ee04ce65e7ae9dc410fb93b5d6a Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 20:47:59 +0300 Subject: [PATCH 28/44] print output if failed --- .../e2e/scenario_endpoint_types_test.go | 8 +++----- .../e2e/scenario_push_notifications_test.go | 6 +++--- maintnotifications/e2e/scenario_stress_test.go | 4 ++-- .../e2e/scenario_timeout_configs_test.go | 6 +++--- maintnotifications/e2e/scenario_tls_configs_test.go | 2 +- maintnotifications/e2e/utils_test.go | 13 +++++++++++++ 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 90e5c1378..f0114e8e9 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -66,7 +66,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { t.Fatalf("[ERROR] Failed to create fault injector: %v", err) } - // Create client factory from configuration factory, err := CreateTestClientFactory("standalone") if err != nil { @@ -74,7 +73,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { } endpointConfig := factory.GetConfig() - defer func() { if dump { p("Pool stats:") @@ -207,7 +205,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) } - p("[FI] Failover action completed for %s: %+v", endpointTest.name, status.Status) + p("[FI] Failover action completed for %s: %s %s", endpointTest.name, status.Status, actionOutputIfFailed(status)) // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) @@ -229,7 +227,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", endpointTest.name, err) } - p("[FI] Migrate action completed for %s: %+v", endpointTest.name, status.Status) + p("[FI] Migrate action completed for %s: %s %s", endpointTest.name, status.Status, actionOutputIfFailed(status)) // Wait for MIGRATING notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { @@ -323,7 +321,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { if err != nil { ef("Bind action failed for %s: %v", endpointTest.name, err) } - p("Bind action completed for %s: %+v", endpointTest.name, bindStatus.Status) + p("Bind action completed for %s: %s %s", endpointTest.name, bindStatus.Status, actionOutputIfFailed(bindStatus)) // Continue traffic for analysis time.Sleep(30 * time.Second) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 8f327185a..95f9b08f1 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -166,7 +166,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed: %v", err) } - p("[FI] Failover action completed: %+v", status.Status) + p("[FI] Failover action completed: %v %s", status.Status, actionOutputIfFailed(status)) p("FAILING_OVER / FAILED_OVER notifications test completed successfully") @@ -203,7 +203,7 @@ func TestPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed: %v", err) } - p("[FI] Migrate action completed: %+v", status.Status) + p("[FI] Migrate action completed: %s %s", status.Status, actionOutputIfFailed(status)) go func() { p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1) @@ -382,7 +382,7 @@ func TestPushNotifications(t *testing.T) { ef("Bind action failed: %v", err) } - p("Bind action completed: %+v", bindStatus.Status) + p("Bind action completed: %s %s", bindStatus.Status, actionOutputIfFailed(bindStatus)) p("MOVING notification test completed successfully") diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index ed14f0c67..95cc0e7f3 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -199,10 +199,10 @@ func TestStressPushNotifications(t *testing.T) { } actionMutex.Lock() - actionResults = append(actionResults, fmt.Sprintf("%s: %+v", actionName, status.Status)) + actionResults = append(actionResults, fmt.Sprintf("%s: %s %s", actionName, status.Status, actionOutputIfFailed(status))) actionMutex.Unlock() - p("[FI] %s action completed: %+v", actionName, status.Status) + p("[FI] %s action completed: %s %s", actionName, status.Status, actionOutputIfFailed(status)) }(action.name, action.action, action.delay) } diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 9b6e346a3..9f90141ff 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -209,7 +209,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Failover action completed for %s: %+v", timeoutTest.name, status.Status) + p("[FI] Failover action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status)) // Continue traffic to observe timeout behavior p("Continuing traffic for %v to observe timeout behavior...", timeoutTest.relaxedTimeout*2) @@ -236,7 +236,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Migrate action completed for %s: %+v", timeoutTest.name, status.Status) + p("[FI] Migrate action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status)) // Wait for MIGRATING notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { @@ -265,7 +265,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err) } - p("[FI] Bind action completed for %s: %+v", timeoutTest.name, status.Status) + p("[FI] Bind action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status)) // waiting for moving notification match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index bcb709aea..02a5dbbfb 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -199,7 +199,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { if err != nil { ef("[FI] Migrate action failed for %s: %v", tlsTest.name, err) } - p("[FI] Migrate action completed for %s: %+v", tlsTest.name, status) + p("[FI] Migrate action completed for %s: %s %s", tlsTest.name, status.Status, actionOutputIfFailed(status)) // Continue traffic for a bit to observe TLS behavior time.Sleep(5 * time.Second) diff --git a/maintnotifications/e2e/utils_test.go b/maintnotifications/e2e/utils_test.go index 66be7b0d4..a60fac89f 100644 --- a/maintnotifications/e2e/utils_test.go +++ b/maintnotifications/e2e/utils_test.go @@ -61,3 +61,16 @@ func printLog(group string, isError bool, format string, args ...interface{}) { args = append([]interface{}{filename, line, ts, group}, args...) fmt.Printf(finalFormat, args...) } + +func actionOutputIfFailed(status *ActionStatusResponse) string { + if status.Status != StatusFailed { + return "" + } + if status.Error != nil { + return fmt.Sprintf("%v", status.Error) + } + if status.Output == nil { + return "" + } + return fmt.Sprintf("%+v", status.Output) +} From 9ed6e2e1df13b36069871a6ffa8b630f1611b102 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 20:56:11 +0300 Subject: [PATCH 29/44] better second and third client checks --- .../e2e/scenario_push_notifications_test.go | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 95f9b08f1..60023e68e 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -266,9 +266,6 @@ func TestPushNotifications(t *testing.T) { errChan := make(chan error, 1) go func() { - var match string - var matchNotif []interface{} - var found bool defer func() { if r := recover(); r != nil { errChan <- fmt.Errorf("goroutine panic: %v", r) @@ -280,6 +277,11 @@ func TestPushNotifications(t *testing.T) { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING") }, 3*time.Minute) commandsRunner.Stop() + if !found { + errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A FIRST CLIENT") + return + } + // once moving is received, start a second client commands runner p("Starting commands on second client") go commandsRunner2.FireCommandsUntilStop(ctx) @@ -289,33 +291,26 @@ func TestPushNotifications(t *testing.T) { // destroy the second client factory.Destroy("push-notification-client-2") }() - // wait for moving on second client - // we know the maxconn is 15, assuming 16/17 was used to init the second client, so connID 18 should be from the second client - // also validate big enough relaxed timeout + p("Waiting for MOVING notification on second client") - matchNotif, found = tracker2.FindOrWaitForNotification("MOVING", 3*time.Minute) - if !found { + matchNotif, fnd := tracker2.FindOrWaitForNotification("MOVING", 3*time.Minute) + if !fnd { errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A SECOND CLIENT") return } else { p("MOVING notification received on second client %v", matchNotif) } - // wait for relaxation of 30m - match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { - return strings.Contains(s, logs2.ApplyingRelaxedTimeoutDueToPostHandoffMessage) && strings.Contains(s, "30m") - }, 3*time.Minute) - if !found { - errChan <- fmt.Errorf("relaxed timeout was not applied within 3 minutes ON A SECOND CLIENT") - return - } else { - p("Relaxed timeout applied on second client") - } // Signal success errChan <- nil }() commandsRunner.FireCommandsUntilStop(ctx) - + // wait for moving on first client + // once the commandRunner stops, it means a waiting + // on the logCollector match has completed and we can proceed + if !found { + ef("MOVING notification was not received within 3 minutes") + } movingData := logs2.ExtractDataFromLogMessage(match) p("MOVING notification received. %v", movingData) seqIDToObserve = int64(movingData["seqID"].(float64)) From f0adb971e2ec8baed3dafcba5b3d75580d823d09 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 21:20:21 +0300 Subject: [PATCH 30/44] output action data if notification is not received --- .../e2e/scenario_push_notifications_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 60023e68e..04a9dc126 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -189,6 +189,14 @@ func TestPushNotifications(t *testing.T) { }() commandsRunner.FireCommandsUntilStop(ctx) if !found { + status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID, + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), + ) + if err != nil { + ef("[FI] Migrate action failed: %v", err) + } + p("[FI] Migrate action completed: %s %s", status.Status, actionOutputIfFailed(status)) ef("MIGRATING notification for migrate action was not received within 60 seconds") } migrateData := logs2.ExtractDataFromLogMessage(match) From db17aa7e4968f7e578b9640093f9f63bf2b34cc1 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Mon, 6 Oct 2025 23:07:30 +0300 Subject: [PATCH 31/44] stop command runner --- maintnotifications/e2e/command_runner_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/maintnotifications/e2e/command_runner_test.go b/maintnotifications/e2e/command_runner_test.go index 7974016a9..b80a434bb 100644 --- a/maintnotifications/e2e/command_runner_test.go +++ b/maintnotifications/e2e/command_runner_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "strings" "sync" "sync/atomic" "time" @@ -88,6 +89,15 @@ func (cr *CommandRunner) FireCommandsUntilStop(ctx context.Context) { cr.operationCount.Add(1) if err != nil { + if err == redis.ErrClosed || strings.Contains(err.Error(), "client is closed") { + select { + case <-cr.stopCh: + return + default: + } + return + } + fmt.Printf("Error: %v\n", err) cr.errorCount.Add(1) From bf3870d7c05a27918b5be69085d5c3a894cf3476 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 13:35:51 +0300 Subject: [PATCH 32/44] database create / delete actions --- maintnotifications/e2e/DATABASE_MANAGEMENT.md | 363 ++++++++++++++++++ maintnotifications/e2e/README_SCENARIOS.md | 21 +- .../examples/database_management_example.go | 205 ++++++++++ maintnotifications/e2e/fault_injector_test.go | 78 ++++ 4 files changed, 664 insertions(+), 3 deletions(-) create mode 100644 maintnotifications/e2e/DATABASE_MANAGEMENT.md create mode 100644 maintnotifications/e2e/examples/database_management_example.go diff --git a/maintnotifications/e2e/DATABASE_MANAGEMENT.md b/maintnotifications/e2e/DATABASE_MANAGEMENT.md new file mode 100644 index 000000000..02dffe762 --- /dev/null +++ b/maintnotifications/e2e/DATABASE_MANAGEMENT.md @@ -0,0 +1,363 @@ +# Database Management with Fault Injector + +This document describes how to use the fault injector's database management endpoints to create and delete Redis databases during E2E testing. + +## Overview + +The fault injector now supports two new endpoints for database management: + +1. **CREATE_DATABASE** - Create a new Redis database with custom configuration +2. **DELETE_DATABASE** - Delete an existing Redis database + +These endpoints are useful for E2E tests that need to dynamically create and destroy databases as part of their test scenarios. + +## Action Types + +### CREATE_DATABASE + +Creates a new Redis database with the specified configuration. + +**Parameters:** +- `cluster_index` (int): The index of the cluster where the database should be created +- `database_config` (object): The database configuration (see structure below) + +**Raises:** +- `CreateDatabaseException`: When database creation fails + +### DELETE_DATABASE + +Deletes an existing Redis database. + +**Parameters:** +- `cluster_index` (int): The index of the cluster containing the database +- `bdb_id` (int): The database ID to delete + +**Raises:** +- `DeleteDatabaseException`: When database deletion fails + +## Database Configuration Structure + +The `database_config` object supports the following fields: + +```go +type DatabaseConfig struct { + Name string `json:"name"` + Port int `json:"port"` + MemorySize int64 `json:"memory_size"` + Replication bool `json:"replication"` + EvictionPolicy string `json:"eviction_policy"` + Sharding bool `json:"sharding"` + AutoUpgrade bool `json:"auto_upgrade"` + ShardsCount int `json:"shards_count"` + ModuleList []DatabaseModule `json:"module_list,omitempty"` + OSSCluster bool `json:"oss_cluster"` + OSSClusterAPIPreferredIPType string `json:"oss_cluster_api_preferred_ip_type,omitempty"` + ProxyPolicy string `json:"proxy_policy,omitempty"` + ShardsPlacement string `json:"shards_placement,omitempty"` + ShardKeyRegex []ShardKeyRegexPattern `json:"shard_key_regex,omitempty"` +} + +type DatabaseModule struct { + ModuleArgs string `json:"module_args"` + ModuleName string `json:"module_name"` +} + +type ShardKeyRegexPattern struct { + Regex string `json:"regex"` +} +``` + +### Example Configuration + +#### Simple Database + +```json +{ + "name": "simple-db", + "port": 12000, + "memory_size": 268435456, + "replication": false, + "eviction_policy": "noeviction", + "sharding": false, + "auto_upgrade": true, + "shards_count": 1, + "oss_cluster": false +} +``` + +#### Clustered Database with Modules + +```json +{ + "name": "ioredis-cluster", + "port": 11112, + "memory_size": 1273741824, + "replication": true, + "eviction_policy": "noeviction", + "sharding": true, + "auto_upgrade": true, + "shards_count": 3, + "module_list": [ + { + "module_args": "", + "module_name": "ReJSON" + }, + { + "module_args": "", + "module_name": "search" + }, + { + "module_args": "", + "module_name": "timeseries" + }, + { + "module_args": "", + "module_name": "bf" + } + ], + "oss_cluster": true, + "oss_cluster_api_preferred_ip_type": "external", + "proxy_policy": "all-master-shards", + "shards_placement": "sparse", + "shard_key_regex": [ + { + "regex": ".*\\{(?.*)\\}.*" + }, + { + "regex": "(?.*)" + } + ] +} +``` + +## Usage Examples + +### Example 1: Create a Simple Database + +```go +ctx := context.Background() +faultInjector := NewFaultInjectorClient("http://127.0.0.1:20324") + +dbConfig := DatabaseConfig{ + Name: "test-db", + Port: 12000, + MemorySize: 268435456, // 256MB + Replication: false, + EvictionPolicy: "noeviction", + Sharding: false, + AutoUpgrade: true, + ShardsCount: 1, + OSSCluster: false, +} + +resp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) +if err != nil { + log.Fatalf("Failed to create database: %v", err) +} + +// Wait for creation to complete +status, err := faultInjector.WaitForAction(ctx, resp.ActionID, + WithMaxWaitTime(5*time.Minute)) +if err != nil { + log.Fatalf("Failed to wait for action: %v", err) +} + +if status.Status == StatusSuccess { + log.Println("Database created successfully!") +} +``` + +### Example 2: Create a Database with Modules + +```go +dbConfig := DatabaseConfig{ + Name: "modules-db", + Port: 12001, + MemorySize: 536870912, // 512MB + Replication: true, + EvictionPolicy: "noeviction", + Sharding: true, + AutoUpgrade: true, + ShardsCount: 3, + ModuleList: []DatabaseModule{ + {ModuleArgs: "", ModuleName: "ReJSON"}, + {ModuleArgs: "", ModuleName: "search"}, + }, + OSSCluster: true, + OSSClusterAPIPreferredIPType: "external", + ProxyPolicy: "all-master-shards", + ShardsPlacement: "sparse", +} + +resp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) +// ... handle response +``` + +### Example 3: Create Database Using a Map + +```go +dbConfigMap := map[string]interface{}{ + "name": "map-db", + "port": 12002, + "memory_size": 268435456, + "replication": false, + "eviction_policy": "volatile-lru", + "sharding": false, + "auto_upgrade": true, + "shards_count": 1, + "oss_cluster": false, +} + +resp, err := faultInjector.CreateDatabaseFromMap(ctx, 0, dbConfigMap) +// ... handle response +``` + +### Example 4: Delete a Database + +```go +clusterIndex := 0 +bdbID := 1 + +resp, err := faultInjector.DeleteDatabase(ctx, clusterIndex, bdbID) +if err != nil { + log.Fatalf("Failed to delete database: %v", err) +} + +status, err := faultInjector.WaitForAction(ctx, resp.ActionID, + WithMaxWaitTime(2*time.Minute)) +if err != nil { + log.Fatalf("Failed to wait for action: %v", err) +} + +if status.Status == StatusSuccess { + log.Println("Database deleted successfully!") +} +``` + +### Example 5: Complete Lifecycle (Create and Delete) + +```go +// Create database +dbConfig := DatabaseConfig{ + Name: "temp-db", + Port: 13000, + MemorySize: 268435456, + Replication: false, + EvictionPolicy: "noeviction", + Sharding: false, + AutoUpgrade: true, + ShardsCount: 1, + OSSCluster: false, +} + +createResp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) +if err != nil { + log.Fatalf("Failed to create database: %v", err) +} + +createStatus, err := faultInjector.WaitForAction(ctx, createResp.ActionID, + WithMaxWaitTime(5*time.Minute)) +if err != nil || createStatus.Status != StatusSuccess { + log.Fatalf("Database creation failed") +} + +// Extract bdb_id from output +var bdbID int +if id, ok := createStatus.Output["bdb_id"].(float64); ok { + bdbID = int(id) +} + +// Use the database for testing... +time.Sleep(10 * time.Second) + +// Delete the database +deleteResp, err := faultInjector.DeleteDatabase(ctx, 0, bdbID) +if err != nil { + log.Fatalf("Failed to delete database: %v", err) +} + +deleteStatus, err := faultInjector.WaitForAction(ctx, deleteResp.ActionID, + WithMaxWaitTime(2*time.Minute)) +if err != nil || deleteStatus.Status != StatusSuccess { + log.Fatalf("Database deletion failed") +} + +log.Println("Database lifecycle completed successfully!") +``` + +## Available Methods + +The `FaultInjectorClient` provides the following methods for database management: + +### CreateDatabase + +```go +func (c *FaultInjectorClient) CreateDatabase( + ctx context.Context, + clusterIndex int, + databaseConfig DatabaseConfig, +) (*ActionResponse, error) +``` + +Creates a new database using a structured `DatabaseConfig` object. + +### CreateDatabaseFromMap + +```go +func (c *FaultInjectorClient) CreateDatabaseFromMap( + ctx context.Context, + clusterIndex int, + databaseConfig map[string]interface{}, +) (*ActionResponse, error) +``` + +Creates a new database using a flexible map configuration. Useful when you need to pass custom or dynamic configurations. + +### DeleteDatabase + +```go +func (c *FaultInjectorClient) DeleteDatabase( + ctx context.Context, + clusterIndex int, + bdbID int, +) (*ActionResponse, error) +``` + +Deletes an existing database by its ID. + +## Testing + +To run the database management E2E tests: + +```bash +# Run all database management tests +go test -tags=e2e -v ./maintnotifications/e2e/ -run TestDatabase + +# Run specific test +go test -tags=e2e -v ./maintnotifications/e2e/ -run TestDatabaseLifecycle +``` + +## Notes + +- Database creation can take several minutes depending on the configuration +- Always use `WaitForAction` to ensure the operation completes before proceeding +- The `bdb_id` returned in the creation output should be used for deletion +- Deleting a non-existent database will result in a failed action status +- Memory sizes are specified in bytes (e.g., 268435456 = 256MB) +- Port numbers should be unique and not conflict with existing databases + +## Common Eviction Policies + +- `noeviction` - Return errors when memory limit is reached +- `allkeys-lru` - Evict any key using LRU algorithm +- `volatile-lru` - Evict keys with TTL using LRU algorithm +- `allkeys-random` - Evict random keys +- `volatile-random` - Evict random keys with TTL +- `volatile-ttl` - Evict keys with TTL, shortest TTL first + +## Common Proxy Policies + +- `all-master-shards` - Route to all master shards +- `all-nodes` - Route to all nodes +- `single-shard` - Route to a single shard + diff --git a/maintnotifications/e2e/README_SCENARIOS.md b/maintnotifications/e2e/README_SCENARIOS.md index 5b778d32b..a9b18de2e 100644 --- a/maintnotifications/e2e/README_SCENARIOS.md +++ b/maintnotifications/e2e/README_SCENARIOS.md @@ -44,7 +44,22 @@ there are three environment variables that need to be set before running the tes - Notification delivery consistency - Handoff behavior per endpoint type -### 3. Timeout Configurations Scenario (`scenario_timeout_configs_test.go`) +### 3. Database Management Scenario (`scenario_database_management_test.go`) +**Dynamic database creation and deletion** +- **Purpose**: Test database lifecycle management via fault injector +- **Features Tested**: CREATE_DATABASE, DELETE_DATABASE endpoints +- **Configuration**: Various database configurations (simple, with modules, clustered) +- **Duration**: ~10 minutes +- **Key Validations**: + - Database creation with different configurations + - Database creation with Redis modules (ReJSON, search, timeseries, bf) + - Database deletion + - Complete lifecycle (create → use → delete) + - Configuration validation + +See [DATABASE_MANAGEMENT.md](DATABASE_MANAGEMENT.md) for detailed documentation on database management endpoints. + +### 4. Timeout Configurations Scenario (`scenario_timeout_configs_test.go`) **Various timeout strategies** - **Purpose**: Test different timeout configurations and their impact - **Features Tested**: Conservative, Aggressive, HighLatency timeouts @@ -58,7 +73,7 @@ there are three environment variables that need to be set before running the tes - Recovery times appropriate for each strategy - Error rates correlate with timeout aggressiveness -### 4. TLS Configurations Scenario (`scenario_tls_configs_test.go`) +### 5. TLS Configurations Scenario (`scenario_tls_configs_test.go`) **Security and encryption testing framework** - **Purpose**: Test push notifications with different TLS configurations - **Features Tested**: NoTLS, TLSInsecure, TLSSecure, TLSMinimal, TLSStrict @@ -71,7 +86,7 @@ there are three environment variables that need to be set before running the tes - Security compliance - **Note**: TLS configuration is handled at the Redis connection config level, not client options level -### 5. Stress Test Scenario (`scenario_stress_test.go`) +### 6. Stress Test Scenario (`scenario_stress_test.go`) **Extreme load and concurrent operations** - **Purpose**: Test system limits and behavior under extreme stress - **Features Tested**: Maximum concurrent operations, multiple clients diff --git a/maintnotifications/e2e/examples/database_management_example.go b/maintnotifications/e2e/examples/database_management_example.go new file mode 100644 index 000000000..e2677441c --- /dev/null +++ b/maintnotifications/e2e/examples/database_management_example.go @@ -0,0 +1,205 @@ +package examples + +import ( + "context" + "fmt" + "time" + + e2e "github.com/redis/go-redis/v9/maintnotifications/e2e" +) + +// ExampleCreateDatabase demonstrates how to create a database using the fault injector +func ExampleCreateDatabase() { + ctx := context.Background() + faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") + + // Example 1: Create a database using the DatabaseConfig struct + dbConfig := e2e.DatabaseConfig{ + Name: "ioredis-cluster", + Port: 11112, + MemorySize: 1273741824, // ~1.2GB + Replication: true, + EvictionPolicy: "noeviction", + Sharding: true, + AutoUpgrade: true, + ShardsCount: 3, + ModuleList: []e2e.DatabaseModule{ + {ModuleArgs: "", ModuleName: "ReJSON"}, + {ModuleArgs: "", ModuleName: "search"}, + {ModuleArgs: "", ModuleName: "timeseries"}, + {ModuleArgs: "", ModuleName: "bf"}, + }, + OSSCluster: true, + OSSClusterAPIPreferredIPType: "external", + ProxyPolicy: "all-master-shards", + ShardsPlacement: "sparse", + ShardKeyRegex: []e2e.ShardKeyRegexPattern{ + {Regex: ".*\\{(?.*)\\}.*"}, + {Regex: "(?.*)"}, + }, + } + + resp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) + if err != nil { + fmt.Printf("Failed to create database: %v\n", err) + return + } + + fmt.Printf("Database creation initiated. Action ID: %s\n", resp.ActionID) + + // Wait for the action to complete + status, err := faultInjector.WaitForAction(ctx, resp.ActionID, + e2e.WithMaxWaitTime(5*time.Minute), + e2e.WithPollInterval(5*time.Second)) + if err != nil { + fmt.Printf("Failed to wait for action: %v\n", err) + return + } + + fmt.Printf("Database creation status: %s\n", status.Status) + if status.Error != nil { + fmt.Printf("Error: %v\n", status.Error) + } +} + +// ExampleCreateDatabaseFromMap demonstrates how to create a database using a map +func ExampleCreateDatabaseFromMap() { + ctx := context.Background() + faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") + + // Example 2: Create a database using a map (more flexible) + dbConfigMap := map[string]interface{}{ + "name": "test-database", + "port": 12000, + "memory_size": 536870912, // 512MB + "replication": false, + "eviction_policy": "volatile-lru", + "sharding": false, + "auto_upgrade": true, + "shards_count": 1, + "oss_cluster": false, + } + + resp, err := faultInjector.CreateDatabaseFromMap(ctx, 0, dbConfigMap) + if err != nil { + fmt.Printf("Failed to create database: %v\n", err) + return + } + + fmt.Printf("Database creation initiated. Action ID: %s\n", resp.ActionID) + + // Wait for the action to complete + status, err := faultInjector.WaitForAction(ctx, resp.ActionID) + if err != nil { + fmt.Printf("Failed to wait for action: %v\n", err) + return + } + + fmt.Printf("Database creation status: %s\n", status.Status) +} + +// ExampleDeleteDatabase demonstrates how to delete a database +func ExampleDeleteDatabase() { + ctx := context.Background() + faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") + + // Delete a database with cluster_index=0 and bdb_id=1 + resp, err := faultInjector.DeleteDatabase(ctx, 0, 1) + if err != nil { + fmt.Printf("Failed to delete database: %v\n", err) + return + } + + fmt.Printf("Database deletion initiated. Action ID: %s\n", resp.ActionID) + + // Wait for the action to complete + status, err := faultInjector.WaitForAction(ctx, resp.ActionID, + e2e.WithMaxWaitTime(2*time.Minute)) + if err != nil { + fmt.Printf("Failed to wait for action: %v\n", err) + return + } + + fmt.Printf("Database deletion status: %s\n", status.Status) + if status.Error != nil { + fmt.Printf("Error: %v\n", status.Error) + } +} + +// ExampleCreateAndDeleteDatabase demonstrates a complete workflow +func ExampleCreateAndDeleteDatabase() { + ctx := context.Background() + faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") + + // Step 1: Create a database + dbConfig := e2e.DatabaseConfig{ + Name: "temp-test-db", + Port: 13000, + MemorySize: 268435456, // 256MB + Replication: false, + EvictionPolicy: "noeviction", + Sharding: false, + AutoUpgrade: true, + ShardsCount: 1, + OSSCluster: false, + } + + createResp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) + if err != nil { + fmt.Printf("Failed to create database: %v\n", err) + return + } + + fmt.Printf("Creating database... Action ID: %s\n", createResp.ActionID) + + createStatus, err := faultInjector.WaitForAction(ctx, createResp.ActionID, + e2e.WithMaxWaitTime(5*time.Minute)) + if err != nil { + fmt.Printf("Failed to wait for database creation: %v\n", err) + return + } + + if createStatus.Status != e2e.StatusSuccess { + fmt.Printf("Database creation failed: %v\n", createStatus.Error) + return + } + + fmt.Printf("Database created successfully!\n") + + // Extract the bdb_id from the output if available + var bdbID int + if output, ok := createStatus.Output["bdb_id"].(float64); ok { + bdbID = int(output) + } else { + // If not in output, you might need to query or use a known ID + bdbID = 1 // Example fallback + } + + // Step 2: Do some work with the database + fmt.Printf("Database is ready for testing (bdb_id: %d)\n", bdbID) + time.Sleep(10 * time.Second) // Simulate some testing + + // Step 3: Delete the database + deleteResp, err := faultInjector.DeleteDatabase(ctx, 0, bdbID) + if err != nil { + fmt.Printf("Failed to delete database: %v\n", err) + return + } + + fmt.Printf("Deleting database... Action ID: %s\n", deleteResp.ActionID) + + deleteStatus, err := faultInjector.WaitForAction(ctx, deleteResp.ActionID, + e2e.WithMaxWaitTime(2*time.Minute)) + if err != nil { + fmt.Printf("Failed to wait for database deletion: %v\n", err) + return + } + + if deleteStatus.Status != e2e.StatusSuccess { + fmt.Printf("Database deletion failed: %v\n", deleteStatus.Error) + return + } + + fmt.Printf("Database deleted successfully!\n") +} + diff --git a/maintnotifications/e2e/fault_injector_test.go b/maintnotifications/e2e/fault_injector_test.go index b1ac92985..cd429608e 100644 --- a/maintnotifications/e2e/fault_injector_test.go +++ b/maintnotifications/e2e/fault_injector_test.go @@ -44,6 +44,10 @@ const ( // Sequence and complex actions ActionSequence ActionType = "sequence_of_actions" ActionExecuteCommand ActionType = "execute_command" + + // Database management actions + ActionDeleteDatabase ActionType = "delete_database" + ActionCreateDatabase ActionType = "create_database" ) // ActionStatus represents the status of an action @@ -350,6 +354,80 @@ func (c *FaultInjectorClient) DisableMaintenanceMode(ctx context.Context, nodeID }) } +// Database Management Actions + +// DatabaseConfig represents the configuration for creating a database +type DatabaseConfig struct { + Name string `json:"name"` + Port int `json:"port"` + MemorySize int64 `json:"memory_size"` + Replication bool `json:"replication"` + EvictionPolicy string `json:"eviction_policy"` + Sharding bool `json:"sharding"` + AutoUpgrade bool `json:"auto_upgrade"` + ShardsCount int `json:"shards_count"` + ModuleList []DatabaseModule `json:"module_list,omitempty"` + OSSCluster bool `json:"oss_cluster"` + OSSClusterAPIPreferredIPType string `json:"oss_cluster_api_preferred_ip_type,omitempty"` + ProxyPolicy string `json:"proxy_policy,omitempty"` + ShardsPlacement string `json:"shards_placement,omitempty"` + ShardKeyRegex []ShardKeyRegexPattern `json:"shard_key_regex,omitempty"` +} + +// DatabaseModule represents a Redis module configuration +type DatabaseModule struct { + ModuleArgs string `json:"module_args"` + ModuleName string `json:"module_name"` +} + +// ShardKeyRegexPattern represents a shard key regex pattern +type ShardKeyRegexPattern struct { + Regex string `json:"regex"` +} + +// DeleteDatabase deletes a database +// Parameters: +// - clusterIndex: The index of the cluster +// - bdbID: The database ID to delete +func (c *FaultInjectorClient) DeleteDatabase(ctx context.Context, clusterIndex int, bdbID int) (*ActionResponse, error) { + return c.TriggerAction(ctx, ActionRequest{ + Type: ActionDeleteDatabase, + Parameters: map[string]interface{}{ + "cluster_index": clusterIndex, + "bdb_id": bdbID, + }, + }) +} + +// CreateDatabase creates a new database +// Parameters: +// - clusterIndex: The index of the cluster +// - databaseConfig: The database configuration +func (c *FaultInjectorClient) CreateDatabase(ctx context.Context, clusterIndex int, databaseConfig DatabaseConfig) (*ActionResponse, error) { + return c.TriggerAction(ctx, ActionRequest{ + Type: ActionCreateDatabase, + Parameters: map[string]interface{}{ + "cluster_index": clusterIndex, + "database_config": databaseConfig, + }, + }) +} + +// CreateDatabaseFromMap creates a new database using a map for configuration +// This is useful when you want to pass a raw configuration map +// Parameters: +// - clusterIndex: The index of the cluster +// - databaseConfig: The database configuration as a map +func (c *FaultInjectorClient) CreateDatabaseFromMap(ctx context.Context, clusterIndex int, databaseConfig map[string]interface{}) (*ActionResponse, error) { + return c.TriggerAction(ctx, ActionRequest{ + Type: ActionCreateDatabase, + Parameters: map[string]interface{}{ + "cluster_index": clusterIndex, + "database_config": databaseConfig, + }, + }) +} + // Complex Actions // ExecuteSequence executes a sequence of actions From 0d4212b9fdd825edbb0795c975e1ffeb52872fcf Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 14:36:04 +0300 Subject: [PATCH 33/44] database create / delete actions used in tests --- maintnotifications/e2e/config_parser_test.go | 361 +++++++++++++++++- maintnotifications/e2e/fault_injector_test.go | 30 +- .../e2e/scenario_endpoint_types_test.go | 28 +- .../e2e/scenario_push_notifications_test.go | 16 +- .../e2e/scenario_stress_test.go | 16 +- .../e2e/scenario_timeout_configs_test.go | 18 +- .../e2e/scenario_tls_configs_test.go | 18 +- 7 files changed, 423 insertions(+), 64 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index ea4901bfd..dcc2729db 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -28,8 +28,8 @@ type DatabaseEndpoint struct { UID string `json:"uid"` } -// DatabaseConfig represents the configuration for a single database -type DatabaseConfig struct { +// EnvDatabaseConfig represents the configuration for a single database +type EnvDatabaseConfig struct { BdbID interface{} `json:"bdb_id,omitempty"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` @@ -39,8 +39,8 @@ type DatabaseConfig struct { Endpoints []string `json:"endpoints"` } -// DatabasesConfig represents the complete configuration file structure -type DatabasesConfig map[string]DatabaseConfig +// EnvDatabasesConfig represents the complete configuration file structure +type EnvDatabasesConfig map[string]EnvDatabaseConfig // EnvConfig represents environment configuration for test scenarios type EnvConfig struct { @@ -80,13 +80,13 @@ func GetEnvConfig() (*EnvConfig, error) { } // GetDatabaseConfigFromEnv reads database configuration from a file -func GetDatabaseConfigFromEnv(filePath string) (DatabasesConfig, error) { +func GetDatabaseConfigFromEnv(filePath string) (EnvDatabasesConfig, error) { fileContent, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("failed to read database config from %s: %w", filePath, err) } - var config DatabasesConfig + var config EnvDatabasesConfig if err := json.Unmarshal(fileContent, &config); err != nil { return nil, fmt.Errorf("failed to parse database config from %s: %w", filePath, err) } @@ -95,8 +95,8 @@ func GetDatabaseConfigFromEnv(filePath string) (DatabasesConfig, error) { } // GetDatabaseConfig gets Redis connection parameters for a specific database -func GetDatabaseConfig(databasesConfig DatabasesConfig, databaseName string) (*RedisConnectionConfig, error) { - var dbConfig DatabaseConfig +func GetDatabaseConfig(databasesConfig EnvDatabasesConfig, databaseName string) (*RedisConnectionConfig, error) { + var dbConfig EnvDatabaseConfig var exists bool if databaseName == "" { @@ -447,6 +447,30 @@ func CreateTestClientFactory(databaseName string) (*ClientFactory, error) { return NewClientFactory(dbConfig), nil } +// CreateTestClientFactoryWithBdbID creates a client factory using a specific bdb_id +// This is useful when you've created a fresh database and want to connect to it +func CreateTestClientFactoryWithBdbID(databaseName string, bdbID int) (*ClientFactory, error) { + envConfig, err := GetEnvConfig() + if err != nil { + return nil, fmt.Errorf("failed to get environment config: %w", err) + } + + databasesConfig, err := GetDatabaseConfigFromEnv(envConfig.RedisEndpointsConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to get database config: %w", err) + } + + dbConfig, err := GetDatabaseConfig(databasesConfig, databaseName) + if err != nil { + return nil, fmt.Errorf("failed to get database config for %s: %w", databaseName, err) + } + + // Override the bdb_id with the newly created database ID + dbConfig.BdbID = bdbID + + return NewClientFactory(dbConfig), nil +} + // CreateTestFaultInjector creates a fault injector client from environment configuration func CreateTestFaultInjector() (*FaultInjectorClient, error) { envConfig, err := GetEnvConfig() @@ -471,3 +495,324 @@ func GetAvailableDatabases(configPath string) ([]string, error) { return databases, nil } + +// ConvertEnvDatabaseConfigToFaultInjectorConfig converts EnvDatabaseConfig to fault injector DatabaseConfig +func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, name string) (DatabaseConfig, error) { + var port int + var dnsName string + + // Extract port and DNS name from raw_endpoints or endpoints + if len(envConfig.RawEndpoints) > 0 { + endpoint := envConfig.RawEndpoints[0] + port = endpoint.Port + dnsName = endpoint.DNSName + } else if len(envConfig.Endpoints) > 0 { + endpointURL, err := url.Parse(envConfig.Endpoints[0]) + if err != nil { + return DatabaseConfig{}, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + dnsName = endpointURL.Hostname() + portStr := endpointURL.Port() + if portStr != "" { + port, err = strconv.Atoi(portStr) + if err != nil { + return DatabaseConfig{}, fmt.Errorf("invalid port: %w", err) + } + } else { + port = 6379 // default + } + } else { + return DatabaseConfig{}, fmt.Errorf("no endpoints found in configuration") + } + + // Build the database config for fault injector + dbConfig := DatabaseConfig{ + Name: name, + Port: port, + MemorySize: 268435456, // 256MB default + Replication: false, + EvictionPolicy: "noeviction", + Sharding: false, + AutoUpgrade: true, + ShardsCount: 1, + OSSCluster: false, + } + + // If we have raw_endpoints with cluster info, configure for cluster + if len(envConfig.RawEndpoints) > 0 { + endpoint := envConfig.RawEndpoints[0] + + // Check if this is a cluster configuration + if endpoint.ProxyPolicy != "" && endpoint.ProxyPolicy != "single" { + dbConfig.OSSCluster = true + dbConfig.Sharding = true + dbConfig.ShardsCount = 3 // default for cluster + dbConfig.ProxyPolicy = endpoint.ProxyPolicy + dbConfig.Replication = true + } + + if endpoint.OSSClusterAPIPreferredIPType != "" { + dbConfig.OSSClusterAPIPreferredIPType = endpoint.OSSClusterAPIPreferredIPType + } + } + + return dbConfig, nil +} + +// TestDatabaseManager manages database lifecycle for tests +type TestDatabaseManager struct { + faultInjector *FaultInjectorClient + clusterIndex int + createdBdbID int + dbConfig DatabaseConfig + t *testing.T +} + +// NewTestDatabaseManager creates a new test database manager +func NewTestDatabaseManager(t *testing.T, faultInjector *FaultInjectorClient, clusterIndex int) *TestDatabaseManager { + return &TestDatabaseManager{ + faultInjector: faultInjector, + clusterIndex: clusterIndex, + t: t, + } +} + +// CreateDatabaseFromEnvConfig creates a database using EnvDatabaseConfig +func (m *TestDatabaseManager) CreateDatabaseFromEnvConfig(ctx context.Context, envConfig EnvDatabaseConfig, name string) (int, error) { + // Convert EnvDatabaseConfig to DatabaseConfig + dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig, name) + if err != nil { + return 0, fmt.Errorf("failed to convert config: %w", err) + } + + m.dbConfig = dbConfig + return m.CreateDatabase(ctx, dbConfig) +} + +// CreateDatabase creates a database and waits for it to be ready +func (m *TestDatabaseManager) CreateDatabase(ctx context.Context, dbConfig DatabaseConfig) (int, error) { + m.t.Logf("Creating database '%s' on port %d...", dbConfig.Name, dbConfig.Port) + + resp, err := m.faultInjector.CreateDatabase(ctx, m.clusterIndex, dbConfig) + if err != nil { + return 0, fmt.Errorf("failed to trigger database creation: %w", err) + } + + m.t.Logf("Database creation triggered. Action ID: %s", resp.ActionID) + + // Wait for creation to complete + status, err := m.faultInjector.WaitForAction(ctx, resp.ActionID, + WithMaxWaitTime(5*time.Minute), + WithPollInterval(5*time.Second)) + if err != nil { + return 0, fmt.Errorf("failed to wait for database creation: %w", err) + } + + if status.Status != StatusSuccess { + return 0, fmt.Errorf("database creation failed: %v", status.Error) + } + + // Extract bdb_id from output + var bdbID int + if status.Output != nil { + if id, ok := status.Output["bdb_id"].(float64); ok { + bdbID = int(id) + } else if resultMap, ok := status.Output["result"].(map[string]interface{}); ok { + if id, ok := resultMap["bdb_id"].(float64); ok { + bdbID = int(id) + } + } + } + + if bdbID == 0 { + return 0, fmt.Errorf("failed to extract bdb_id from creation output") + } + + m.createdBdbID = bdbID + m.t.Logf("Database created successfully with bdb_id: %d", bdbID) + + return bdbID, nil +} + +// DeleteDatabase deletes the created database +func (m *TestDatabaseManager) DeleteDatabase(ctx context.Context) error { + if m.createdBdbID == 0 { + return fmt.Errorf("no database to delete (bdb_id is 0)") + } + + m.t.Logf("Deleting database with bdb_id: %d...", m.createdBdbID) + + resp, err := m.faultInjector.DeleteDatabase(ctx, m.clusterIndex, m.createdBdbID) + if err != nil { + return fmt.Errorf("failed to trigger database deletion: %w", err) + } + + m.t.Logf("Database deletion triggered. Action ID: %s", resp.ActionID) + + // Wait for deletion to complete + status, err := m.faultInjector.WaitForAction(ctx, resp.ActionID, + WithMaxWaitTime(2*time.Minute), + WithPollInterval(3*time.Second)) + if err != nil { + return fmt.Errorf("failed to wait for database deletion: %w", err) + } + + if status.Status != StatusSuccess { + return fmt.Errorf("database deletion failed: %v", status.Error) + } + + m.t.Logf("Database deleted successfully") + m.createdBdbID = 0 + + return nil +} + +// GetBdbID returns the created database ID +func (m *TestDatabaseManager) GetBdbID() int { + return m.createdBdbID +} + +// Cleanup ensures the database is deleted (safe to call multiple times) +func (m *TestDatabaseManager) Cleanup(ctx context.Context) { + if m.createdBdbID != 0 { + if err := m.DeleteDatabase(ctx); err != nil { + m.t.Logf("Warning: Failed to cleanup database: %v", err) + } + } +} + +// SetupTestDatabaseFromEnv creates a database from environment config and returns a cleanup function +// Usage: +// cleanup := SetupTestDatabaseFromEnv(t, ctx, "my-test-db") +// defer cleanup() +func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName string) (bdbID int, cleanup func()) { + // Get environment config + envConfig, err := GetEnvConfig() + if err != nil { + t.Fatalf("Failed to get environment config: %v", err) + } + + // Get database config from environment + databasesConfig, err := GetDatabaseConfigFromEnv(envConfig.RedisEndpointsConfigPath) + if err != nil { + t.Fatalf("Failed to get database config: %v", err) + } + + // Get the specific database config + var envDbConfig EnvDatabaseConfig + var exists bool + if databaseName == "" { + // Get first database if no name provided + for _, config := range databasesConfig { + envDbConfig = config + exists = true + break + } + } else { + envDbConfig, exists = databasesConfig[databaseName] + } + + if !exists { + t.Fatalf("Database %s not found in configuration", databaseName) + } + + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("Failed to create fault injector: %v", err) + } + + // Create database manager + dbManager := NewTestDatabaseManager(t, faultInjector, 0) + + // Create the database + testDBName := fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix()) + bdbID, err = dbManager.CreateDatabaseFromEnvConfig(ctx, envDbConfig, testDBName) + if err != nil { + t.Fatalf("Failed to create test database: %v", err) + } + + // Return cleanup function + cleanup = func() { + dbManager.Cleanup(ctx) + } + + return bdbID, cleanup +} + +// SetupTestDatabaseWithConfig creates a database with custom config and returns a cleanup function +// Usage: +// bdbID, cleanup := SetupTestDatabaseWithConfig(t, ctx, dbConfig) +// defer cleanup() +func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig DatabaseConfig) (bdbID int, cleanup func()) { + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("Failed to create fault injector: %v", err) + } + + // Create database manager + dbManager := NewTestDatabaseManager(t, faultInjector, 0) + + // Create the database + bdbID, err = dbManager.CreateDatabase(ctx, dbConfig) + if err != nil { + t.Fatalf("Failed to create test database: %v", err) + } + + // Return cleanup function + cleanup = func() { + dbManager.Cleanup(ctx) + } + + return bdbID, cleanup +} + +// SetupTestDatabaseAndFactory creates a database from environment config and returns both bdbID, factory, and cleanup function +// This is the recommended way to setup tests as it ensures the client factory connects to the newly created database +// Usage: +// bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") +// defer cleanup() +func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName string) (bdbID int, factory *ClientFactory, cleanup func()) { + // Create the database + bdbID, cleanupDB := SetupTestDatabaseFromEnv(t, ctx, databaseName) + + // Create client factory that connects to the new database + factory, err := CreateTestClientFactoryWithBdbID(databaseName, bdbID) + if err != nil { + cleanupDB() // Clean up the database if factory creation fails + t.Fatalf("Failed to create client factory for new database: %v", err) + } + + // Combined cleanup function + cleanup = func() { + factory.DestroyAll() + cleanupDB() + } + + return bdbID, factory, cleanup +} + +// SetupTestDatabaseAndFactoryWithConfig creates a database with custom config and returns both bdbID, factory, and cleanup function +// Usage: +// bdbID, factory, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig) +// defer cleanup() +func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, databaseName string, dbConfig DatabaseConfig) (bdbID int, factory *ClientFactory, cleanup func()) { + // Create the database + bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) + + // Create client factory that connects to the new database + factory, err := CreateTestClientFactoryWithBdbID(databaseName, bdbID) + if err != nil { + cleanupDB() // Clean up the database if factory creation fails + t.Fatalf("Failed to create client factory for new database: %v", err) + } + + // Combined cleanup function + cleanup = func() { + factory.DestroyAll() + cleanupDB() + } + + return bdbID, factory, cleanup +} diff --git a/maintnotifications/e2e/fault_injector_test.go b/maintnotifications/e2e/fault_injector_test.go index cd429608e..64c8da1e8 100644 --- a/maintnotifications/e2e/fault_injector_test.go +++ b/maintnotifications/e2e/fault_injector_test.go @@ -356,22 +356,22 @@ func (c *FaultInjectorClient) DisableMaintenanceMode(ctx context.Context, nodeID // Database Management Actions -// DatabaseConfig represents the configuration for creating a database +// EnvDatabaseConfig represents the configuration for creating a database type DatabaseConfig struct { - Name string `json:"name"` - Port int `json:"port"` - MemorySize int64 `json:"memory_size"` - Replication bool `json:"replication"` - EvictionPolicy string `json:"eviction_policy"` - Sharding bool `json:"sharding"` - AutoUpgrade bool `json:"auto_upgrade"` - ShardsCount int `json:"shards_count"` - ModuleList []DatabaseModule `json:"module_list,omitempty"` - OSSCluster bool `json:"oss_cluster"` - OSSClusterAPIPreferredIPType string `json:"oss_cluster_api_preferred_ip_type,omitempty"` - ProxyPolicy string `json:"proxy_policy,omitempty"` - ShardsPlacement string `json:"shards_placement,omitempty"` - ShardKeyRegex []ShardKeyRegexPattern `json:"shard_key_regex,omitempty"` + Name string `json:"name"` + Port int `json:"port"` + MemorySize int64 `json:"memory_size"` + Replication bool `json:"replication"` + EvictionPolicy string `json:"eviction_policy"` + Sharding bool `json:"sharding"` + AutoUpgrade bool `json:"auto_upgrade"` + ShardsCount int `json:"shards_count"` + ModuleList []DatabaseModule `json:"module_list,omitempty"` + OSSCluster bool `json:"oss_cluster"` + OSSClusterAPIPreferredIPType string `json:"oss_cluster_api_preferred_ip_type,omitempty"` + ProxyPolicy string `json:"proxy_policy,omitempty"` + ShardsPlacement string `json:"shards_placement,omitempty"` + ShardKeyRegex []ShardKeyRegexPattern `json:"shard_key_regex,omitempty"` } // DatabaseModule represents a Redis module configuration diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index f0114e8e9..d36acbe25 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -21,17 +21,19 @@ func TestEndpointTypesPushNotifications(t *testing.T) { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) defer cancel() + // Setup: Create fresh database and client factory for this test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[ENDPOINT-TYPES] Created test database with bdb_id: %d", bdbID) + + // Wait for database to be fully ready + time.Sleep(10 * time.Second) + var dump = true var errorsDetected = false - var p = func(format string, args ...interface{}) { - format = "[%s][ENDPOINT-TYPES] " + format - ts := time.Now().Format("15:04:05.000") - args = append([]interface{}{ts}, args...) - t.Logf(format, args...) - } // Test different endpoint types endpointTypes := []struct { @@ -66,19 +68,15 @@ func TestEndpointTypesPushNotifications(t *testing.T) { t.Fatalf("[ERROR] Failed to create fault injector: %v", err) } - // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") - if err != nil { - t.Skipf("Enterprise cluster not available, skipping endpoint types test: %v", err) - } + // Get endpoint config from factory (now connected to new database) endpointConfig := factory.GetConfig() defer func() { if dump { - p("Pool stats:") + fmt.Println("Pool stats:") factory.PrintPoolStats(t) } - factory.DestroyAll() + // Note: factory.DestroyAll() is called by cleanup() function }() // Test each endpoint type @@ -377,5 +375,5 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }) } - p("All endpoint types tested successfully") + t.Log("All endpoint types tested successfully") } diff --git a/maintnotifications/e2e/scenario_push_notifications_test.go b/maintnotifications/e2e/scenario_push_notifications_test.go index 04a9dc126..7666ed0c4 100644 --- a/maintnotifications/e2e/scenario_push_notifications_test.go +++ b/maintnotifications/e2e/scenario_push_notifications_test.go @@ -19,9 +19,17 @@ func TestPushNotifications(t *testing.T) { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() + // Setup: Create fresh database and client factory for this test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[PUSH-NOTIFICATIONS] Created test database with bdb_id: %d", bdbID) + + // Wait for database to be fully ready + time.Sleep(10 * time.Second) + var dump = true var seqIDToObserve int64 var connIDToObserve uint64 @@ -51,11 +59,7 @@ func TestPushNotifications(t *testing.T) { logCollector.Clear() }() - // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") - if err != nil { - t.Skipf("[PUSH-NOTIFICATIONS][SKIP] Enterprise cluster not available, skipping push notification tests: %v", err) - } + // Get endpoint config from factory (now connected to new database) endpointConfig := factory.GetConfig() // Create fault injector diff --git a/maintnotifications/e2e/scenario_stress_test.go b/maintnotifications/e2e/scenario_stress_test.go index 95cc0e7f3..2eea14448 100644 --- a/maintnotifications/e2e/scenario_stress_test.go +++ b/maintnotifications/e2e/scenario_stress_test.go @@ -19,9 +19,17 @@ func TestStressPushNotifications(t *testing.T) { t.Skip("[STRESS][SKIP] Scenario tests require E2E_SCENARIO_TESTS=true") } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Minute) defer cancel() + // Setup: Create fresh database and client factory for this test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[STRESS] Created test database with bdb_id: %d", bdbID) + + // Wait for database to be fully ready + time.Sleep(10 * time.Second) + var dump = true var errorsDetected = false @@ -44,11 +52,7 @@ func TestStressPushNotifications(t *testing.T) { logCollector.Clear() }() - // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") - if err != nil { - t.Skipf("[STRESS][SKIP] Enterprise cluster not available, skipping stress test: %v", err) - } + // Get endpoint config from factory (now connected to new database) endpointConfig := factory.GetConfig() // Create fault injector diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index 9f90141ff..dca47877f 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -19,9 +19,17 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() + // Setup: Create fresh database and client factory for this test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[TIMEOUT-CONFIGS] Created test database with bdb_id: %d", bdbID) + + // Wait for database to be fully ready + time.Sleep(10 * time.Second) + var dump = true var errorsDetected = false @@ -74,11 +82,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { logCollector.Clear() }() - // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") - if err != nil { - t.Skipf("[TIMEOUT-CONFIGS][SKIP] Enterprise cluster not available, skipping timeout configs test: %v", err) - } + // Get endpoint config from factory (now connected to new database) endpointConfig := factory.GetConfig() // Create fault injector @@ -92,7 +96,7 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { p("Pool stats:") factory.PrintPoolStats(t) } - factory.DestroyAll() + // Note: factory.DestroyAll() is called by cleanup() function }() // Test each timeout configuration diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index 02a5dbbfb..b451633e4 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -20,9 +20,17 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true") } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) defer cancel() + // Setup: Create fresh database and client factory for this test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[TLS-CONFIGS] Created test database with bdb_id: %d", bdbID) + + // Wait for database to be fully ready + time.Sleep(10 * time.Second) + var dump = true var errorsDetected = false var p = func(format string, args ...interface{}) { @@ -70,11 +78,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { logCollector.Clear() }() - // Create client factory from configuration - factory, err := CreateTestClientFactory("standalone") - if err != nil { - t.Skipf("[TLS-CONFIGS][SKIP] Enterprise cluster not available, skipping TLS configs test: %v", err) - } + // Get endpoint config from factory (now connected to new database) endpointConfig := factory.GetConfig() // Create fault injector @@ -88,7 +92,7 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { p("Pool stats:") factory.PrintPoolStats(t) } - factory.DestroyAll() + // Note: factory.DestroyAll() is called by cleanup() function }() // Test each TLS configuration From 820a33d7c008d62bb7860fb4029a734c53fa88cc Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 14:51:33 +0300 Subject: [PATCH 34/44] fix import --- maintnotifications/e2e/config_parser_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index dcc2729db..f1fafd5a9 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "crypto/tls" "encoding/json" "fmt" From 37a7f8dea858ce438ea5a2f01870627ba61a21b0 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 14:55:28 +0300 Subject: [PATCH 35/44] remove example --- maintnotifications/e2e/config_parser_test.go | 20 +- .../examples/database_management_example.go | 205 ------------------ 2 files changed, 12 insertions(+), 213 deletions(-) delete mode 100644 maintnotifications/e2e/examples/database_management_example.go diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index f1fafd5a9..05c964022 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -684,8 +684,9 @@ func (m *TestDatabaseManager) Cleanup(ctx context.Context) { // SetupTestDatabaseFromEnv creates a database from environment config and returns a cleanup function // Usage: -// cleanup := SetupTestDatabaseFromEnv(t, ctx, "my-test-db") -// defer cleanup() +// +// cleanup := SetupTestDatabaseFromEnv(t, ctx, "my-test-db") +// defer cleanup() func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName string) (bdbID int, cleanup func()) { // Get environment config envConfig, err := GetEnvConfig() @@ -743,8 +744,9 @@ func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName st // SetupTestDatabaseWithConfig creates a database with custom config and returns a cleanup function // Usage: -// bdbID, cleanup := SetupTestDatabaseWithConfig(t, ctx, dbConfig) -// defer cleanup() +// +// bdbID, cleanup := SetupTestDatabaseWithConfig(t, ctx, dbConfig) +// defer cleanup() func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig DatabaseConfig) (bdbID int, cleanup func()) { // Create fault injector faultInjector, err := CreateTestFaultInjector() @@ -772,8 +774,9 @@ func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig Dat // SetupTestDatabaseAndFactory creates a database from environment config and returns both bdbID, factory, and cleanup function // This is the recommended way to setup tests as it ensures the client factory connects to the newly created database // Usage: -// bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") -// defer cleanup() +// +// bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") +// defer cleanup() func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName string) (bdbID int, factory *ClientFactory, cleanup func()) { // Create the database bdbID, cleanupDB := SetupTestDatabaseFromEnv(t, ctx, databaseName) @@ -796,8 +799,9 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName // SetupTestDatabaseAndFactoryWithConfig creates a database with custom config and returns both bdbID, factory, and cleanup function // Usage: -// bdbID, factory, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig) -// defer cleanup() +// +// bdbID, factory, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig) +// defer cleanup() func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, databaseName string, dbConfig DatabaseConfig) (bdbID int, factory *ClientFactory, cleanup func()) { // Create the database bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) diff --git a/maintnotifications/e2e/examples/database_management_example.go b/maintnotifications/e2e/examples/database_management_example.go deleted file mode 100644 index e2677441c..000000000 --- a/maintnotifications/e2e/examples/database_management_example.go +++ /dev/null @@ -1,205 +0,0 @@ -package examples - -import ( - "context" - "fmt" - "time" - - e2e "github.com/redis/go-redis/v9/maintnotifications/e2e" -) - -// ExampleCreateDatabase demonstrates how to create a database using the fault injector -func ExampleCreateDatabase() { - ctx := context.Background() - faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") - - // Example 1: Create a database using the DatabaseConfig struct - dbConfig := e2e.DatabaseConfig{ - Name: "ioredis-cluster", - Port: 11112, - MemorySize: 1273741824, // ~1.2GB - Replication: true, - EvictionPolicy: "noeviction", - Sharding: true, - AutoUpgrade: true, - ShardsCount: 3, - ModuleList: []e2e.DatabaseModule{ - {ModuleArgs: "", ModuleName: "ReJSON"}, - {ModuleArgs: "", ModuleName: "search"}, - {ModuleArgs: "", ModuleName: "timeseries"}, - {ModuleArgs: "", ModuleName: "bf"}, - }, - OSSCluster: true, - OSSClusterAPIPreferredIPType: "external", - ProxyPolicy: "all-master-shards", - ShardsPlacement: "sparse", - ShardKeyRegex: []e2e.ShardKeyRegexPattern{ - {Regex: ".*\\{(?.*)\\}.*"}, - {Regex: "(?.*)"}, - }, - } - - resp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) - if err != nil { - fmt.Printf("Failed to create database: %v\n", err) - return - } - - fmt.Printf("Database creation initiated. Action ID: %s\n", resp.ActionID) - - // Wait for the action to complete - status, err := faultInjector.WaitForAction(ctx, resp.ActionID, - e2e.WithMaxWaitTime(5*time.Minute), - e2e.WithPollInterval(5*time.Second)) - if err != nil { - fmt.Printf("Failed to wait for action: %v\n", err) - return - } - - fmt.Printf("Database creation status: %s\n", status.Status) - if status.Error != nil { - fmt.Printf("Error: %v\n", status.Error) - } -} - -// ExampleCreateDatabaseFromMap demonstrates how to create a database using a map -func ExampleCreateDatabaseFromMap() { - ctx := context.Background() - faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") - - // Example 2: Create a database using a map (more flexible) - dbConfigMap := map[string]interface{}{ - "name": "test-database", - "port": 12000, - "memory_size": 536870912, // 512MB - "replication": false, - "eviction_policy": "volatile-lru", - "sharding": false, - "auto_upgrade": true, - "shards_count": 1, - "oss_cluster": false, - } - - resp, err := faultInjector.CreateDatabaseFromMap(ctx, 0, dbConfigMap) - if err != nil { - fmt.Printf("Failed to create database: %v\n", err) - return - } - - fmt.Printf("Database creation initiated. Action ID: %s\n", resp.ActionID) - - // Wait for the action to complete - status, err := faultInjector.WaitForAction(ctx, resp.ActionID) - if err != nil { - fmt.Printf("Failed to wait for action: %v\n", err) - return - } - - fmt.Printf("Database creation status: %s\n", status.Status) -} - -// ExampleDeleteDatabase demonstrates how to delete a database -func ExampleDeleteDatabase() { - ctx := context.Background() - faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") - - // Delete a database with cluster_index=0 and bdb_id=1 - resp, err := faultInjector.DeleteDatabase(ctx, 0, 1) - if err != nil { - fmt.Printf("Failed to delete database: %v\n", err) - return - } - - fmt.Printf("Database deletion initiated. Action ID: %s\n", resp.ActionID) - - // Wait for the action to complete - status, err := faultInjector.WaitForAction(ctx, resp.ActionID, - e2e.WithMaxWaitTime(2*time.Minute)) - if err != nil { - fmt.Printf("Failed to wait for action: %v\n", err) - return - } - - fmt.Printf("Database deletion status: %s\n", status.Status) - if status.Error != nil { - fmt.Printf("Error: %v\n", status.Error) - } -} - -// ExampleCreateAndDeleteDatabase demonstrates a complete workflow -func ExampleCreateAndDeleteDatabase() { - ctx := context.Background() - faultInjector := e2e.NewFaultInjectorClient("http://127.0.0.1:20324") - - // Step 1: Create a database - dbConfig := e2e.DatabaseConfig{ - Name: "temp-test-db", - Port: 13000, - MemorySize: 268435456, // 256MB - Replication: false, - EvictionPolicy: "noeviction", - Sharding: false, - AutoUpgrade: true, - ShardsCount: 1, - OSSCluster: false, - } - - createResp, err := faultInjector.CreateDatabase(ctx, 0, dbConfig) - if err != nil { - fmt.Printf("Failed to create database: %v\n", err) - return - } - - fmt.Printf("Creating database... Action ID: %s\n", createResp.ActionID) - - createStatus, err := faultInjector.WaitForAction(ctx, createResp.ActionID, - e2e.WithMaxWaitTime(5*time.Minute)) - if err != nil { - fmt.Printf("Failed to wait for database creation: %v\n", err) - return - } - - if createStatus.Status != e2e.StatusSuccess { - fmt.Printf("Database creation failed: %v\n", createStatus.Error) - return - } - - fmt.Printf("Database created successfully!\n") - - // Extract the bdb_id from the output if available - var bdbID int - if output, ok := createStatus.Output["bdb_id"].(float64); ok { - bdbID = int(output) - } else { - // If not in output, you might need to query or use a known ID - bdbID = 1 // Example fallback - } - - // Step 2: Do some work with the database - fmt.Printf("Database is ready for testing (bdb_id: %d)\n", bdbID) - time.Sleep(10 * time.Second) // Simulate some testing - - // Step 3: Delete the database - deleteResp, err := faultInjector.DeleteDatabase(ctx, 0, bdbID) - if err != nil { - fmt.Printf("Failed to delete database: %v\n", err) - return - } - - fmt.Printf("Deleting database... Action ID: %s\n", deleteResp.ActionID) - - deleteStatus, err := faultInjector.WaitForAction(ctx, deleteResp.ActionID, - e2e.WithMaxWaitTime(2*time.Minute)) - if err != nil { - fmt.Printf("Failed to wait for database deletion: %v\n", err) - return - } - - if deleteStatus.Status != e2e.StatusSuccess { - fmt.Printf("Database deletion failed: %v\n", deleteStatus.Error) - return - } - - fmt.Printf("Database deleted successfully!\n") -} - From 13c50921863f9dcb9b3de01f53f1a99dbbe1c0d3 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 14:56:30 +0300 Subject: [PATCH 36/44] remove unused var --- maintnotifications/e2e/config_parser_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index 05c964022..140d4069a 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -500,19 +500,16 @@ func GetAvailableDatabases(configPath string) ([]string, error) { // ConvertEnvDatabaseConfigToFaultInjectorConfig converts EnvDatabaseConfig to fault injector DatabaseConfig func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, name string) (DatabaseConfig, error) { var port int - var dnsName string // Extract port and DNS name from raw_endpoints or endpoints if len(envConfig.RawEndpoints) > 0 { endpoint := envConfig.RawEndpoints[0] port = endpoint.Port - dnsName = endpoint.DNSName } else if len(envConfig.Endpoints) > 0 { endpointURL, err := url.Parse(envConfig.Endpoints[0]) if err != nil { return DatabaseConfig{}, fmt.Errorf("failed to parse endpoint URL: %w", err) } - dnsName = endpointURL.Hostname() portStr := endpointURL.Port() if portStr != "" { port, err = strconv.Atoi(portStr) From c74156f539c980e143c4b46d7e71f7464285066b Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 15:43:09 +0300 Subject: [PATCH 37/44] use different port than the one in env --- maintnotifications/e2e/config_parser_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index 140d4069a..1b687be78 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "math/rand" "net/url" "os" "strconv" @@ -517,16 +518,18 @@ func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, return DatabaseConfig{}, fmt.Errorf("invalid port: %w", err) } } else { - port = 6379 // default + port = 6379 * 2 // default*2 } } else { return DatabaseConfig{}, fmt.Errorf("no endpoints found in configuration") } + randomPortOffset := 1 + rand.Intn(10) // Random port offset to avoid conflicts + // Build the database config for fault injector dbConfig := DatabaseConfig{ Name: name, - Port: port, + Port: port + randomPortOffset, MemorySize: 268435456, // 256MB default Replication: false, EvictionPolicy: "noeviction", From f1a4f46a00d31dd601f7385d170528a59053f9ca Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 16:03:54 +0300 Subject: [PATCH 38/44] wait for action to get the response --- .../e2e/scenario_endpoint_types_test.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index d36acbe25..b8b2e617e 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -157,7 +157,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Test failover with this endpoint type - p("Testing failover with %s endpoint type...", endpointTest.name) + p("Testing failover with %s endpoint type on database %s[bdb_id:%s]...", endpointTest.name, endpointConfig.Name, endpointConfig.BdbID) failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ @@ -173,6 +173,16 @@ func TestEndpointTypesPushNotifications(t *testing.T) { commandsRunner.FireCommandsUntilStop(ctx) }() + // Wait for failover to complete + status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, + WithMaxWaitTime(240*time.Second), + WithPollInterval(2*time.Second), + ) + if err != nil { + ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) + } + p("[FI] Failover action completed for %s: %s %s", endpointTest.name, status.Status, actionOutputIfFailed(status)) + // Wait for FAILING_OVER notification match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool { return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILING_OVER") @@ -195,16 +205,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { failedOverData := logs2.ExtractDataFromLogMessage(match) p("FAILED_OVER notification received for %s. %v", endpointTest.name, failedOverData) - // Wait for failover to complete - status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID, - WithMaxWaitTime(240*time.Second), - WithPollInterval(2*time.Second), - ) - if err != nil { - ef("[FI] Failover action failed for %s: %v", endpointTest.name, err) - } - p("[FI] Failover action completed for %s: %s %s", endpointTest.name, status.Status, actionOutputIfFailed(status)) - // Test migration with this endpoint type p("Testing migration with %s endpoint type...", endpointTest.name) migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ From ecd679bfe40ebb5b88baefcc2eab658f93c1adbd Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 16:41:48 +0300 Subject: [PATCH 39/44] fix output --- maintnotifications/e2e/fault_injector_test.go | 1 + maintnotifications/e2e/scenario_endpoint_types_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/maintnotifications/e2e/fault_injector_test.go b/maintnotifications/e2e/fault_injector_test.go index 64c8da1e8..fbb5d4a5f 100644 --- a/maintnotifications/e2e/fault_injector_test.go +++ b/maintnotifications/e2e/fault_injector_test.go @@ -124,6 +124,7 @@ func (c *FaultInjectorClient) ListActions(ctx context.Context) ([]ActionType, er // TriggerAction triggers a specific action func (c *FaultInjectorClient) TriggerAction(ctx context.Context, action ActionRequest) (*ActionResponse, error) { var response ActionResponse + fmt.Printf("[FI] Triggering action: %+v\n", action) err := c.request(ctx, "POST", "/action", action, &response) return &response, err } diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index b8b2e617e..0f3e6c73b 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -157,7 +157,7 @@ func TestEndpointTypesPushNotifications(t *testing.T) { }() // Test failover with this endpoint type - p("Testing failover with %s endpoint type on database %s[bdb_id:%s]...", endpointTest.name, endpointConfig.Name, endpointConfig.BdbID) + p("Testing failover with %s endpoint type on database [bdb_id:%s]...", endpointTest.name, endpointConfig.BdbID) failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{ Type: "failover", Parameters: map[string]interface{}{ From 38f7b5e121001fee4d79effd1ea8c7b73544f446 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 17:14:11 +0300 Subject: [PATCH 40/44] fix create db config --- maintnotifications/e2e/config_parser_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index 1b687be78..a030d9e06 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -528,15 +528,17 @@ func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, // Build the database config for fault injector dbConfig := DatabaseConfig{ - Name: name, - Port: port + randomPortOffset, - MemorySize: 268435456, // 256MB default - Replication: false, - EvictionPolicy: "noeviction", - Sharding: false, - AutoUpgrade: true, - ShardsCount: 1, - OSSCluster: false, + Name: name, + Port: port + randomPortOffset, + MemorySize: 268435456, // 256MB default + Replication: true, + EvictionPolicy: "noeviction", + ProxyPolicy: "single", + AutoUpgrade: true, + Sharding: true, + ShardsCount: 2, + ShardsPlacement: "dense", + OSSCluster: false, } // If we have raw_endpoints with cluster info, configure for cluster From 37c8c956ad312f38f944a3c4d0175273526e02b5 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 17:31:02 +0300 Subject: [PATCH 41/44] fix create db config --- maintnotifications/e2e/config_parser_test.go | 32 ++++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index a030d9e06..fe645af41 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -527,18 +527,30 @@ func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, randomPortOffset := 1 + rand.Intn(10) // Random port offset to avoid conflicts // Build the database config for fault injector + // TODO: Make this configurable + // IT is the defaults for a sharded database at the moment dbConfig := DatabaseConfig{ - Name: name, - Port: port + randomPortOffset, - MemorySize: 268435456, // 256MB default - Replication: true, - EvictionPolicy: "noeviction", - ProxyPolicy: "single", - AutoUpgrade: true, - Sharding: true, - ShardsCount: 2, + Name: name, + Port: port + randomPortOffset, + MemorySize: 268435456, // 256MB default + Replication: true, + EvictionPolicy: "noeviction", + ProxyPolicy: "single", + AutoUpgrade: true, + Sharding: true, + ShardsCount: 2, + ShardKeyRegex: []ShardKeyRegexPattern{ + {Regex: ".*\\{(?.*)\\}.*"}, + {Regex: "(?.*)"}, + }, ShardsPlacement: "dense", - OSSCluster: false, + ModuleList: []DatabaseModule{ + {ModuleArgs: "", ModuleName: "ReJSON"}, + {ModuleArgs: "", ModuleName: "search"}, + {ModuleArgs: "", ModuleName: "timeseries"}, + {ModuleArgs: "", ModuleName: "bf"}, + }, + OSSCluster: false, } // If we have raw_endpoints with cluster info, configure for cluster From d89e3f4e82948606d5440df77da9046f7c38852f Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 18:10:07 +0300 Subject: [PATCH 42/44] use new database for client --- maintnotifications/e2e/config_parser_test.go | 166 +++++++++++++++++-- 1 file changed, 155 insertions(+), 11 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index fe645af41..78e6b89f3 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -312,6 +312,7 @@ func (cf *ClientFactory) Create(key string, options *CreateClientOptions) (redis } } + fmt.Printf("Creating single client with options: %+v\n", clientOptions) client = redis.NewClient(clientOptions) } @@ -792,16 +793,91 @@ func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig Dat // bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") // defer cleanup() func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName string) (bdbID int, factory *ClientFactory, cleanup func()) { - // Create the database - bdbID, cleanupDB := SetupTestDatabaseFromEnv(t, ctx, databaseName) + // Get environment config + envConfig, err := GetEnvConfig() + if err != nil { + t.Fatalf("Failed to get environment config: %v", err) + } + + // Get database config from environment + databasesConfig, err := GetDatabaseConfigFromEnv(envConfig.RedisEndpointsConfigPath) + if err != nil { + t.Fatalf("Failed to get database config: %v", err) + } + + // Get the specific database config + var envDbConfig EnvDatabaseConfig + var exists bool + if databaseName == "" { + // Get first database if no name provided + for _, config := range databasesConfig { + envDbConfig = config + exists = true + break + } + } else { + envDbConfig, exists = databasesConfig[databaseName] + } - // Create client factory that connects to the new database - factory, err := CreateTestClientFactoryWithBdbID(databaseName, bdbID) + if !exists { + t.Fatalf("Database %s not found in configuration", databaseName) + } + + // Convert to DatabaseConfig to get the port + dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfig(envDbConfig, fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix())) if err != nil { - cleanupDB() // Clean up the database if factory creation fails - t.Fatalf("Failed to create client factory for new database: %v", err) + t.Fatalf("Failed to convert config: %v", err) + } + + // Create the database + bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) + + // Update the environment config with the new database's connection details + // The new database uses the port we specified in dbConfig + newEnvConfig := envDbConfig + newEnvConfig.BdbID = bdbID + + // Update endpoints to point to the new database's port + // Extract host from original endpoints + var host string + var scheme string + if len(envDbConfig.Endpoints) > 0 { + // Parse the first endpoint to get host and scheme + endpoint := envDbConfig.Endpoints[0] + if strings.HasPrefix(endpoint, "redis://") { + scheme = "redis" + host = strings.TrimPrefix(endpoint, "redis://") + } else if strings.HasPrefix(endpoint, "rediss://") { + scheme = "rediss" + host = strings.TrimPrefix(endpoint, "rediss://") + } + // Remove port from host if present + if colonIdx := strings.Index(host, ":"); colonIdx != -1 { + host = host[:colonIdx] + } + } + + // If we couldn't extract host, use localhost as fallback + if host == "" { + host = "localhost" + } + if scheme == "" { + if envDbConfig.TLS { + scheme = "rediss" + } else { + scheme = "redis" + } } + // Construct new endpoint with the new database's port + newEndpoint := fmt.Sprintf("%s://%s:%d", scheme, host, dbConfig.Port) + newEnvConfig.Endpoints = []string{newEndpoint} + + t.Logf("New database endpoint: %s (bdb_id: %d)", newEndpoint, bdbID) + + // Create client factory with the updated config + factory = NewClientFactory(newEnvConfig) + // Combined cleanup function cleanup = func() { factory.DestroyAll() @@ -817,16 +893,84 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName // bdbID, factory, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig) // defer cleanup() func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, databaseName string, dbConfig DatabaseConfig) (bdbID int, factory *ClientFactory, cleanup func()) { + // Get environment config to use as template for connection details + envConfig, err := GetEnvConfig() + if err != nil { + t.Fatalf("Failed to get environment config: %v", err) + } + + // Get database config from environment + databasesConfig, err := GetDatabaseConfigFromEnv(envConfig.RedisEndpointsConfigPath) + if err != nil { + t.Fatalf("Failed to get database config: %v", err) + } + + // Get the specific database config as template + var envDbConfig EnvDatabaseConfig + var exists bool + if databaseName == "" { + // Get first database if no name provided + for _, config := range databasesConfig { + envDbConfig = config + exists = true + break + } + } else { + envDbConfig, exists = databasesConfig[databaseName] + } + + if !exists { + t.Fatalf("Database %s not found in configuration", databaseName) + } + // Create the database bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) - // Create client factory that connects to the new database - factory, err := CreateTestClientFactoryWithBdbID(databaseName, bdbID) - if err != nil { - cleanupDB() // Clean up the database if factory creation fails - t.Fatalf("Failed to create client factory for new database: %v", err) + // Update the environment config with the new database's connection details + newEnvConfig := envDbConfig + newEnvConfig.BdbID = bdbID + + // Update endpoints to point to the new database's port + // Extract host from original endpoints + var host string + var scheme string + if len(envDbConfig.Endpoints) > 0 { + // Parse the first endpoint to get host and scheme + endpoint := envDbConfig.Endpoints[0] + if strings.HasPrefix(endpoint, "redis://") { + scheme = "redis" + host = strings.TrimPrefix(endpoint, "redis://") + } else if strings.HasPrefix(endpoint, "rediss://") { + scheme = "rediss" + host = strings.TrimPrefix(endpoint, "rediss://") + } + // Remove port from host if present + if colonIdx := strings.Index(host, ":"); colonIdx != -1 { + host = host[:colonIdx] + } } + // If we couldn't extract host, use localhost as fallback + if host == "" { + host = "localhost" + } + if scheme == "" { + if envDbConfig.TLS { + scheme = "rediss" + } else { + scheme = "redis" + } + } + + // Construct new endpoint with the new database's port + newEndpoint := fmt.Sprintf("%s://%s:%d", scheme, host, dbConfig.Port) + newEnvConfig.Endpoints = []string{newEndpoint} + + t.Logf("New database endpoint: %s (bdb_id: %d)", newEndpoint, bdbID) + + // Create client factory with the updated config + factory = NewClientFactory(newEnvConfig) + // Combined cleanup function cleanup = func() { factory.DestroyAll() From 8b1b8b6dc40585552602b5dbbabd9d4575562671 Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 18:52:23 +0300 Subject: [PATCH 43/44] fix create db config --- maintnotifications/e2e/config_parser_test.go | 327 ++++++++++++++----- 1 file changed, 243 insertions(+), 84 deletions(-) diff --git a/maintnotifications/e2e/config_parser_test.go b/maintnotifications/e2e/config_parser_test.go index 78e6b89f3..6f1bc3252 100644 --- a/maintnotifications/e2e/config_parser_test.go +++ b/maintnotifications/e2e/config_parser_test.go @@ -181,6 +181,73 @@ func GetDatabaseConfig(databasesConfig EnvDatabasesConfig, databaseName string) }, nil } +// ConvertEnvDatabaseConfigToRedisConnectionConfig converts EnvDatabaseConfig to RedisConnectionConfig +func ConvertEnvDatabaseConfigToRedisConnectionConfig(dbConfig EnvDatabaseConfig) (*RedisConnectionConfig, error) { + // Parse connection details from endpoints or raw_endpoints + var host string + var port int + + if len(dbConfig.RawEndpoints) > 0 { + // Use raw_endpoints if available (for more complex configurations) + endpoint := dbConfig.RawEndpoints[0] // Use the first endpoint + host = endpoint.DNSName + port = endpoint.Port + } else if len(dbConfig.Endpoints) > 0 { + // Parse from endpoints URLs + endpointURL, err := url.Parse(dbConfig.Endpoints[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint URL %s: %w", dbConfig.Endpoints[0], err) + } + + host = endpointURL.Hostname() + portStr := endpointURL.Port() + if portStr == "" { + // Default ports based on scheme + switch endpointURL.Scheme { + case "redis": + port = 6379 + case "rediss": + port = 6380 + default: + port = 6379 + } + } else { + port, err = strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("invalid port in endpoint URL %s: %w", dbConfig.Endpoints[0], err) + } + } + + // Override TLS setting based on scheme if not explicitly set + if endpointURL.Scheme == "rediss" { + dbConfig.TLS = true + } + } else { + return nil, fmt.Errorf("no endpoints found in database configuration") + } + + var bdbId int + switch dbConfig.BdbID.(type) { + case int: + bdbId = dbConfig.BdbID.(int) + case float64: + bdbId = int(dbConfig.BdbID.(float64)) + case string: + bdbId, _ = strconv.Atoi(dbConfig.BdbID.(string)) + } + + return &RedisConnectionConfig{ + Host: host, + Port: port, + Username: dbConfig.Username, + Password: dbConfig.Password, + TLS: dbConfig.TLS, + BdbID: bdbId, + CertificatesLocation: dbConfig.CertificatesLocation, + Endpoints: dbConfig.Endpoints, + }, nil +} + // ClientFactory manages Redis client creation and lifecycle type ClientFactory struct { config *RedisConnectionConfig @@ -606,6 +673,7 @@ func (m *TestDatabaseManager) CreateDatabaseFromEnvConfig(ctx context.Context, e } // CreateDatabase creates a database and waits for it to be ready +// Returns the bdb_id of the created database func (m *TestDatabaseManager) CreateDatabase(ctx context.Context, dbConfig DatabaseConfig) (int, error) { m.t.Logf("Creating database '%s' on port %d...", dbConfig.Name, dbConfig.Port) @@ -650,6 +718,122 @@ func (m *TestDatabaseManager) CreateDatabase(ctx context.Context, dbConfig Datab return bdbID, nil } +// CreateDatabaseAndGetConfig creates a database and returns both the bdb_id and the full connection config from the fault injector response +// This includes endpoints, username, password, TLS settings, and raw_endpoints +func (m *TestDatabaseManager) CreateDatabaseAndGetConfig(ctx context.Context, dbConfig DatabaseConfig) (int, EnvDatabaseConfig, error) { + m.t.Logf("Creating database '%s' on port %d...", dbConfig.Name, dbConfig.Port) + + resp, err := m.faultInjector.CreateDatabase(ctx, m.clusterIndex, dbConfig) + if err != nil { + return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to trigger database creation: %w", err) + } + + m.t.Logf("Database creation triggered. Action ID: %s", resp.ActionID) + + // Wait for creation to complete + status, err := m.faultInjector.WaitForAction(ctx, resp.ActionID, + WithMaxWaitTime(5*time.Minute), + WithPollInterval(5*time.Second)) + if err != nil { + return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to wait for database creation: %w", err) + } + + if status.Status != StatusSuccess { + return 0, EnvDatabaseConfig{}, fmt.Errorf("database creation failed: %v", status.Error) + } + + // Extract database configuration from output + var envConfig EnvDatabaseConfig + if status.Output == nil { + return 0, EnvDatabaseConfig{}, fmt.Errorf("no output in creation response") + } + + // Extract bdb_id + var bdbID int + if id, ok := status.Output["bdb_id"].(float64); ok { + bdbID = int(id) + envConfig.BdbID = bdbID + } else { + return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to extract bdb_id from creation output") + } + + // Extract username + if username, ok := status.Output["username"].(string); ok { + envConfig.Username = username + } + + // Extract password + if password, ok := status.Output["password"].(string); ok { + envConfig.Password = password + } + + // Extract TLS setting + if tls, ok := status.Output["tls"].(bool); ok { + envConfig.TLS = tls + } + + // Extract endpoints + if endpoints, ok := status.Output["endpoints"].([]interface{}); ok { + envConfig.Endpoints = make([]string, 0, len(endpoints)) + for _, ep := range endpoints { + if epStr, ok := ep.(string); ok { + envConfig.Endpoints = append(envConfig.Endpoints, epStr) + } + } + } + + // Extract raw_endpoints + if rawEndpoints, ok := status.Output["raw_endpoints"].([]interface{}); ok { + envConfig.RawEndpoints = make([]DatabaseEndpoint, 0, len(rawEndpoints)) + for _, rawEp := range rawEndpoints { + if rawEpMap, ok := rawEp.(map[string]interface{}); ok { + var dbEndpoint DatabaseEndpoint + + // Extract addr + if addr, ok := rawEpMap["addr"].([]interface{}); ok { + dbEndpoint.Addr = make([]string, 0, len(addr)) + for _, a := range addr { + if aStr, ok := a.(string); ok { + dbEndpoint.Addr = append(dbEndpoint.Addr, aStr) + } + } + } + + // Extract other fields + if addrType, ok := rawEpMap["addr_type"].(string); ok { + dbEndpoint.AddrType = addrType + } + if dnsName, ok := rawEpMap["dns_name"].(string); ok { + dbEndpoint.DNSName = dnsName + } + if preferredEndpointType, ok := rawEpMap["oss_cluster_api_preferred_endpoint_type"].(string); ok { + dbEndpoint.OSSClusterAPIPreferredEndpointType = preferredEndpointType + } + if preferredIPType, ok := rawEpMap["oss_cluster_api_preferred_ip_type"].(string); ok { + dbEndpoint.OSSClusterAPIPreferredIPType = preferredIPType + } + if port, ok := rawEpMap["port"].(float64); ok { + dbEndpoint.Port = int(port) + } + if proxyPolicy, ok := rawEpMap["proxy_policy"].(string); ok { + dbEndpoint.ProxyPolicy = proxyPolicy + } + if uid, ok := rawEpMap["uid"].(string); ok { + dbEndpoint.UID = uid + } + + envConfig.RawEndpoints = append(envConfig.RawEndpoints, dbEndpoint) + } + } + } + + m.createdBdbID = bdbID + m.t.Logf("Database created successfully with bdb_id: %d", bdbID) + m.t.Logf("Database endpoints: %v", envConfig.Endpoints) + + return bdbID, envConfig, nil +} + // DeleteDatabase deletes the created database func (m *TestDatabaseManager) DeleteDatabase(ctx context.Context) error { if m.createdBdbID == 0 { @@ -823,65 +1007,52 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName t.Fatalf("Database %s not found in configuration", databaseName) } - // Convert to DatabaseConfig to get the port + // Convert to DatabaseConfig dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfig(envDbConfig, fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix())) if err != nil { t.Fatalf("Failed to convert config: %v", err) } - // Create the database - bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("Failed to create fault injector: %v", err) + } - // Update the environment config with the new database's connection details - // The new database uses the port we specified in dbConfig - newEnvConfig := envDbConfig - newEnvConfig.BdbID = bdbID + // Create database manager + dbManager := NewTestDatabaseManager(t, faultInjector, 0) - // Update endpoints to point to the new database's port - // Extract host from original endpoints - var host string - var scheme string - if len(envDbConfig.Endpoints) > 0 { - // Parse the first endpoint to get host and scheme - endpoint := envDbConfig.Endpoints[0] - if strings.HasPrefix(endpoint, "redis://") { - scheme = "redis" - host = strings.TrimPrefix(endpoint, "redis://") - } else if strings.HasPrefix(endpoint, "rediss://") { - scheme = "rediss" - host = strings.TrimPrefix(endpoint, "rediss://") - } - // Remove port from host if present - if colonIdx := strings.Index(host, ":"); colonIdx != -1 { - host = host[:colonIdx] - } + // Create the database and get the actual connection config from fault injector + bdbID, newEnvConfig, err := dbManager.CreateDatabaseAndGetConfig(ctx, dbConfig) + if err != nil { + t.Fatalf("Failed to create test database: %v", err) } - // If we couldn't extract host, use localhost as fallback - if host == "" { - host = "localhost" - } - if scheme == "" { - if envDbConfig.TLS { - scheme = "rediss" - } else { - scheme = "redis" - } - } + t.Logf("Database created successfully:") + t.Logf(" bdb_id: %d", bdbID) + t.Logf(" endpoints: %v", newEnvConfig.Endpoints) + t.Logf(" username: %s", newEnvConfig.Username) + t.Logf(" TLS: %v", newEnvConfig.TLS) - // Construct new endpoint with the new database's port - newEndpoint := fmt.Sprintf("%s://%s:%d", scheme, host, dbConfig.Port) - newEnvConfig.Endpoints = []string{newEndpoint} + // Use certificate location from original config if not provided by fault injector + if newEnvConfig.CertificatesLocation == "" && envDbConfig.CertificatesLocation != "" { + newEnvConfig.CertificatesLocation = envDbConfig.CertificatesLocation + } - t.Logf("New database endpoint: %s (bdb_id: %d)", newEndpoint, bdbID) + // Convert EnvDatabaseConfig to RedisConnectionConfig + redisConfig, err := ConvertEnvDatabaseConfigToRedisConnectionConfig(newEnvConfig) + if err != nil { + dbManager.Cleanup(ctx) + t.Fatalf("Failed to convert database config: %v", err) + } - // Create client factory with the updated config - factory = NewClientFactory(newEnvConfig) + // Create client factory with the actual config from fault injector + factory = NewClientFactory(redisConfig) // Combined cleanup function cleanup = func() { factory.DestroyAll() - cleanupDB() + dbManager.Cleanup(ctx) } return bdbID, factory, cleanup @@ -923,58 +1094,46 @@ func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, da t.Fatalf("Database %s not found in configuration", databaseName) } - // Create the database - bdbID, cleanupDB := SetupTestDatabaseWithConfig(t, ctx, dbConfig) + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("Failed to create fault injector: %v", err) + } - // Update the environment config with the new database's connection details - newEnvConfig := envDbConfig - newEnvConfig.BdbID = bdbID + // Create database manager + dbManager := NewTestDatabaseManager(t, faultInjector, 0) - // Update endpoints to point to the new database's port - // Extract host from original endpoints - var host string - var scheme string - if len(envDbConfig.Endpoints) > 0 { - // Parse the first endpoint to get host and scheme - endpoint := envDbConfig.Endpoints[0] - if strings.HasPrefix(endpoint, "redis://") { - scheme = "redis" - host = strings.TrimPrefix(endpoint, "redis://") - } else if strings.HasPrefix(endpoint, "rediss://") { - scheme = "rediss" - host = strings.TrimPrefix(endpoint, "rediss://") - } - // Remove port from host if present - if colonIdx := strings.Index(host, ":"); colonIdx != -1 { - host = host[:colonIdx] - } + // Create the database and get the actual connection config from fault injector + bdbID, newEnvConfig, err := dbManager.CreateDatabaseAndGetConfig(ctx, dbConfig) + if err != nil { + t.Fatalf("Failed to create test database: %v", err) } - // If we couldn't extract host, use localhost as fallback - if host == "" { - host = "localhost" - } - if scheme == "" { - if envDbConfig.TLS { - scheme = "rediss" - } else { - scheme = "redis" - } - } + t.Logf("Database created successfully:") + t.Logf(" bdb_id: %d", bdbID) + t.Logf(" endpoints: %v", newEnvConfig.Endpoints) + t.Logf(" username: %s", newEnvConfig.Username) + t.Logf(" TLS: %v", newEnvConfig.TLS) - // Construct new endpoint with the new database's port - newEndpoint := fmt.Sprintf("%s://%s:%d", scheme, host, dbConfig.Port) - newEnvConfig.Endpoints = []string{newEndpoint} + // Use certificate location from original config if not provided by fault injector + if newEnvConfig.CertificatesLocation == "" && envDbConfig.CertificatesLocation != "" { + newEnvConfig.CertificatesLocation = envDbConfig.CertificatesLocation + } - t.Logf("New database endpoint: %s (bdb_id: %d)", newEndpoint, bdbID) + // Convert EnvDatabaseConfig to RedisConnectionConfig + redisConfig, err := ConvertEnvDatabaseConfigToRedisConnectionConfig(newEnvConfig) + if err != nil { + dbManager.Cleanup(ctx) + t.Fatalf("Failed to convert database config: %v", err) + } - // Create client factory with the updated config - factory = NewClientFactory(newEnvConfig) + // Create client factory with the actual config from fault injector + factory = NewClientFactory(redisConfig) // Combined cleanup function cleanup = func() { factory.DestroyAll() - cleanupDB() + dbManager.Cleanup(ctx) } return bdbID, factory, cleanup From dbe717beded4ca50abc6e5b4dd546367d2bc4f5f Mon Sep 17 00:00:00 2001 From: Nedyalko Dyakov Date: Tue, 7 Oct 2025 19:43:26 +0300 Subject: [PATCH 44/44] db per scenario --- .../e2e/scenario_endpoint_types_test.go | 45 ++++++++---------- .../e2e/scenario_timeout_configs_test.go | 46 +++++++++---------- .../e2e/scenario_tls_configs_test.go | 46 +++++++++---------- 3 files changed, 62 insertions(+), 75 deletions(-) diff --git a/maintnotifications/e2e/scenario_endpoint_types_test.go b/maintnotifications/e2e/scenario_endpoint_types_test.go index 0f3e6c73b..57bd9439f 100644 --- a/maintnotifications/e2e/scenario_endpoint_types_test.go +++ b/maintnotifications/e2e/scenario_endpoint_types_test.go @@ -24,14 +24,6 @@ func TestEndpointTypesPushNotifications(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) defer cancel() - // Setup: Create fresh database and client factory for this test - bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") - defer cleanup() - t.Logf("[ENDPOINT-TYPES] Created test database with bdb_id: %d", bdbID) - - // Wait for database to be fully ready - time.Sleep(10 * time.Second) - var dump = true var errorsDetected = false @@ -62,26 +54,29 @@ func TestEndpointTypesPushNotifications(t *testing.T) { logCollector.Clear() }() - // Create fault injector - faultInjector, err := CreateTestFaultInjector() - if err != nil { - t.Fatalf("[ERROR] Failed to create fault injector: %v", err) - } + // Test each endpoint type with its own fresh database + for _, endpointTest := range endpointTypes { + t.Run(endpointTest.name, func(t *testing.T) { + // Setup: Create fresh database and client factory for THIS endpoint type test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[ENDPOINT-TYPES-%s] Created test database with bdb_id: %d", endpointTest.name, bdbID) - // Get endpoint config from factory (now connected to new database) - endpointConfig := factory.GetConfig() + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) + } - defer func() { - if dump { - fmt.Println("Pool stats:") - factory.PrintPoolStats(t) - } - // Note: factory.DestroyAll() is called by cleanup() function - }() + // Get endpoint config from factory (now connected to new database) + endpointConfig := factory.GetConfig() - // Test each endpoint type - for _, endpointTest := range endpointTypes { - t.Run(endpointTest.name, func(t *testing.T) { + defer func() { + if dump { + fmt.Println("Pool stats:") + factory.PrintPoolStats(t) + } + }() // Clear logs between endpoint type tests logCollector.Clear() // reset errors detected flag diff --git a/maintnotifications/e2e/scenario_timeout_configs_test.go b/maintnotifications/e2e/scenario_timeout_configs_test.go index dca47877f..ae7fcdb0d 100644 --- a/maintnotifications/e2e/scenario_timeout_configs_test.go +++ b/maintnotifications/e2e/scenario_timeout_configs_test.go @@ -22,14 +22,6 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - // Setup: Create fresh database and client factory for this test - bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") - defer cleanup() - t.Logf("[TIMEOUT-CONFIGS] Created test database with bdb_id: %d", bdbID) - - // Wait for database to be fully ready - time.Sleep(10 * time.Second) - var dump = true var errorsDetected = false @@ -82,26 +74,30 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) { logCollector.Clear() }() - // Get endpoint config from factory (now connected to new database) - endpointConfig := factory.GetConfig() + // Test each timeout configuration with its own fresh database + for _, timeoutTest := range timeoutConfigs { + t.Run(timeoutTest.name, func(t *testing.T) { + // Setup: Create fresh database and client factory for THIS timeout config test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[TIMEOUT-CONFIGS-%s] Created test database with bdb_id: %d", timeoutTest.name, bdbID) - // Create fault injector - faultInjector, err := CreateTestFaultInjector() - if err != nil { - t.Fatalf("[ERROR] Failed to create fault injector: %v", err) - } + // Get endpoint config from factory (now connected to new database) + endpointConfig := factory.GetConfig() - defer func() { - if dump { - p("Pool stats:") - factory.PrintPoolStats(t) - } - // Note: factory.DestroyAll() is called by cleanup() function - }() + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) + } + + defer func() { + if dump { + p("Pool stats:") + factory.PrintPoolStats(t) + } + }() - // Test each timeout configuration - for _, timeoutTest := range timeoutConfigs { - t.Run(timeoutTest.name, func(t *testing.T) { errorsDetected = false var ef = func(format string, args ...interface{}) { printLog("TIMEOUT-CONFIGS", true, format, args...) diff --git a/maintnotifications/e2e/scenario_tls_configs_test.go b/maintnotifications/e2e/scenario_tls_configs_test.go index b451633e4..243ea3b7c 100644 --- a/maintnotifications/e2e/scenario_tls_configs_test.go +++ b/maintnotifications/e2e/scenario_tls_configs_test.go @@ -23,14 +23,6 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) defer cancel() - // Setup: Create fresh database and client factory for this test - bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") - defer cleanup() - t.Logf("[TLS-CONFIGS] Created test database with bdb_id: %d", bdbID) - - // Wait for database to be fully ready - time.Sleep(10 * time.Second) - var dump = true var errorsDetected = false var p = func(format string, args ...interface{}) { @@ -78,26 +70,30 @@ func ТestTLSConfigurationsPushNotifications(t *testing.T) { logCollector.Clear() }() - // Get endpoint config from factory (now connected to new database) - endpointConfig := factory.GetConfig() + // Test each TLS configuration with its own fresh database + for _, tlsTest := range tlsConfigs { + t.Run(tlsTest.name, func(t *testing.T) { + // Setup: Create fresh database and client factory for THIS TLS config test + bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone") + defer cleanup() + t.Logf("[TLS-CONFIGS-%s] Created test database with bdb_id: %d", tlsTest.name, bdbID) - // Create fault injector - faultInjector, err := CreateTestFaultInjector() - if err != nil { - t.Fatalf("[ERROR] Failed to create fault injector: %v", err) - } + // Get endpoint config from factory (now connected to new database) + endpointConfig := factory.GetConfig() - defer func() { - if dump { - p("Pool stats:") - factory.PrintPoolStats(t) - } - // Note: factory.DestroyAll() is called by cleanup() function - }() + // Create fault injector + faultInjector, err := CreateTestFaultInjector() + if err != nil { + t.Fatalf("[ERROR] Failed to create fault injector: %v", err) + } + + defer func() { + if dump { + p("Pool stats:") + factory.PrintPoolStats(t) + } + }() - // Test each TLS configuration - for _, tlsTest := range tlsConfigs { - t.Run(tlsTest.name, func(t *testing.T) { errorsDetected = false var ef = func(format string, args ...interface{}) { printLog("TLS-CONFIGS", true, format, args...)