Skip to content

Commit 9fd3b9b

Browse files
vazoisvazois
committed
wip; restructuring cluster tests to reduce CI duration
1 parent dcd7d06 commit 9fd3b9b

File tree

4 files changed

+162
-190
lines changed

4 files changed

+162
-190
lines changed

‎test/Garnet.test.cluster/ClusterManagementTests.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -930,5 +930,100 @@ public void ClusterCheckpointUpgradeFrom([Values] InstanceType instanceType)
930930
ClassicAssert.AreEqual(value, dbValue);
931931
}
932932
}
933+
934+
[Test, Order(16)]
935+
[CancelAfter(30_000)]
936+
[TestCase(false)]
937+
[TestCase(true)]
938+
public async Task ClusterReplicationObjectCollectTest(bool useManualCollect, CancellationToken cancellationToken)
939+
{
940+
var replica_count = 1;// Per primary
941+
var primary_count = 1;
942+
var nodes_count = primary_count + (primary_count * replica_count);
943+
var primaryNodeIndex = 0;
944+
var replicaNodeIndex = 1;
945+
946+
context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: false, asyncReplay: false, expiredObjectCollectionFrequencySecs: !useManualCollect ? 100 : 0);
947+
context.CreateConnection(useTLS: false);
948+
949+
var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex);
950+
var replicaServer = context.clusterTestUtils.GetServer(replicaNodeIndex);
951+
952+
// Setup cluster
953+
context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger);
954+
context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger);
955+
context.clusterTestUtils.SetConfigEpoch(replicaNodeIndex, replicaNodeIndex + 1, logger: context.logger);
956+
context.clusterTestUtils.Meet(primaryNodeIndex, replicaNodeIndex, logger: context.logger);
957+
context.clusterTestUtils.WaitUntilNodeIsKnown(primaryNodeIndex, replicaNodeIndex, logger: context.logger);
958+
context.clusterTestUtils.WaitUntilNodeIsKnown(replicaNodeIndex, primaryNodeIndex, logger: context.logger);
959+
960+
var db = context.clusterTestUtils.GetDatabase();
961+
HashEntry[] elements = [new HashEntry("field1", "hello"), new HashEntry("field2", "world"), new HashEntry("field3", "value3"), new HashEntry("field4", "value4"), new HashEntry("field5", "value5"), new HashEntry("field6", "value6")];
962+
// Attach replica
963+
var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryNodeIndex, logger: context.logger);
964+
ClassicAssert.AreEqual("OK", resp);
965+
966+
// Execute first hash set workload
967+
var hashSetKey = "myhash";
968+
ExecuteHashSet(hashSetKey, elements);
969+
await Task.Delay(3000).ConfigureAwait(false);
970+
ManualCollect();
971+
ValidateHashSet(hashSetKey);
972+
973+
// Execute second hash set workload
974+
hashSetKey = "myhash2";
975+
ExecuteHashSet(hashSetKey, elements);
976+
await Task.Delay(3000).ConfigureAwait(false);
977+
ManualCollect();
978+
ValidateHashSet(hashSetKey);
979+
980+
void ExecuteHashSet(string key, HashEntry[] elements)
981+
{
982+
db.HashSet(key, elements);
983+
984+
var result = primaryServer.Execute("HPEXPIRE", key, "500", "FIELDS", "2", "field1", "field2");
985+
var results = (RedisResult[])result;
986+
ClassicAssert.AreEqual(2, results.Length);
987+
ClassicAssert.AreEqual(1, (long)results[0]);
988+
ClassicAssert.AreEqual(1, (long)results[1]);
989+
990+
result = primaryServer.Execute("HPEXPIRE", key, "500", "FIELDS", "2", "field3", "field4");
991+
results = (RedisResult[])result;
992+
ClassicAssert.AreEqual(2, results.Length);
993+
ClassicAssert.AreEqual(1, (long)results[0]);
994+
ClassicAssert.AreEqual(1, (long)results[1]);
995+
}
996+
997+
void ValidateHashSet(string key)
998+
{
999+
// Check expected result at primary
1000+
var expectedFieldsAndValues = elements.AsSpan().Slice(4).ToArray()
1001+
.SelectMany(e => new[] { e.Name.ToString(), e.Value.ToString() })
1002+
.ToArray();
1003+
var fields = (string[])primaryServer.Execute("HGETALL", [key]);
1004+
ClassicAssert.AreEqual(4, fields.Length);
1005+
ClassicAssert.AreEqual(expectedFieldsAndValues, fields);
1006+
1007+
// Wait to ensure sync
1008+
context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger, cancellationToken);
1009+
1010+
// Check if replica is caught up
1011+
fields = (string[])replicaServer.Execute("HGETALL", [key]);
1012+
ClassicAssert.AreEqual(4, fields.Length);
1013+
ClassicAssert.AreEqual(expectedFieldsAndValues, fields);
1014+
}
1015+
1016+
void ManualCollect()
1017+
{
1018+
if (useManualCollect)
1019+
{
1020+
Assert.Throws<RedisServerException>(() => replicaServer.Execute("HCOLLECT", ["*"], CommandFlags.NoRedirect),
1021+
$"Expected exception was not thrown");
1022+
1023+
resp = (string)primaryServer.Execute("HCOLLECT", ["*"]);
1024+
ClassicAssert.AreEqual("OK", resp);
1025+
}
1026+
}
1027+
}
9331028
}
9341029
}

‎test/Garnet.test.cluster/ClusterNegativeTests.cs‎

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ public void ClusterCheckpointAcquireTest([Values] bool fastAofTruncate, [Values]
291291
}
292292
}
293293

294-
[Test, Order(6), CancelAfter(testTimeout)]
294+
[Test, Order(7), CancelAfter(testTimeout)]
295295
public void ClusterReplicaAttachIntenseWrite(CancellationToken cancellationToken)
296296
{
297297
var primaryIndex = 0;
@@ -369,7 +369,7 @@ public void ClusterReplicaAttachIntenseWrite(CancellationToken cancellationToken
369369
}
370370
}
371371

372-
[Test, Order(6), CancelAfter(testTimeout)]
372+
[Test, Order(8), CancelAfter(testTimeout)]
373373
public void ClusterFailedToAddAofSyncTask()
374374
{
375375
var primaryIndex = 0;
@@ -419,7 +419,7 @@ public void ClusterFailedToAddAofSyncTask()
419419
context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, replicaIndex);
420420
}
421421

422-
[Test, Order(6), CancelAfter(testTimeout)]
422+
[Test, Order(9), CancelAfter(testTimeout)]
423423
public void ClusterReplicaSyncTimeoutTest()
424424
{
425425
var primaryIndex = 0;
@@ -468,7 +468,7 @@ public void ClusterReplicaSyncTimeoutTest()
468468
}
469469
#endif
470470

471-
[Test, CancelAfter(60_000)]
471+
[Test, Order(10), CancelAfter(60_000)]
472472
public async Task ClusterParallelFailoverOnDistinctShards(CancellationToken cancellationToken)
473473
{
474474
var nodes_count = 4;
@@ -523,8 +523,7 @@ public async Task ClusterParallelFailoverOnDistinctShards(CancellationToken canc
523523
}
524524
}
525525

526-
527-
[Test, CancelAfter(60_000)]
526+
[Test, Order(11), CancelAfter(60_000)]
528527
public void ClusterMeetFromReplica(CancellationToken cancellationToken)
529528
{
530529
var nodes_count = 3;
@@ -559,5 +558,66 @@ public void ClusterMeetFromReplica(CancellationToken cancellationToken)
559558
Assert.That(nodes_count, Is.EqualTo(context.clusterTestUtils.ClusterNodes(i).Nodes.Count));
560559
}
561560
}
561+
562+
[Test, Order(12)]
563+
[Category("REPLICATION")]
564+
public void ClusterDontKnowReplicaFailTest([Values] bool useReplicaOf)
565+
{
566+
var replica_count = 1;// Per primary
567+
var primary_count = 1;
568+
var nodes_count = primary_count + (primary_count * replica_count);
569+
ClassicAssert.IsTrue(primary_count > 0);
570+
context.CreateInstances(nodes_count, disableObjects: true, FastAofTruncate: false, OnDemandCheckpoint: false, CommitFrequencyMs: -1, enableAOF: true, useTLS: false, asyncReplay: false);
571+
context.CreateConnection(useTLS: false);
572+
573+
var primaryNodeIndex = 0;
574+
var replicaNodeIndex = 1;
575+
ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddDelSlotsRange(0, [(0, 16383)], true, context.logger));
576+
context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, 1, logger: context.logger);
577+
context.clusterTestUtils.SetConfigEpoch(replicaNodeIndex, 2, logger: context.logger);
578+
579+
var primaryId = context.clusterTestUtils.ClusterMyId(primaryNodeIndex, logger: context.logger);
580+
string resp;
581+
if (!useReplicaOf)
582+
resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryId, failEx: false, logger: context.logger);
583+
else
584+
resp = context.clusterTestUtils.ReplicaOf(replicaNodeIndex, primaryNodeIndex, failEx: false, logger: context.logger);
585+
ClassicAssert.IsTrue(resp.StartsWith("PRIMARY-ERR"));
586+
}
587+
588+
[Test, Order(13)]
589+
[Category("REPLICATION")]
590+
public void ClusterReplicateFails()
591+
{
592+
const string UserName = "temp-user";
593+
const string Password = "temp-password";
594+
595+
const string ClusterUserName = "cluster-user";
596+
const string ClusterPassword = "cluster-password";
597+
598+
// Setup a cluster (mimicking the style in which this bug was first found)
599+
ServerCredential clusterCreds = new(ClusterUserName, ClusterPassword, IsAdmin: true, UsedForClusterAuth: true, IsClearText: true);
600+
ServerCredential userCreds = new(UserName, Password, IsAdmin: true, UsedForClusterAuth: false, IsClearText: true);
601+
602+
context.GenerateCredentials([userCreds, clusterCreds]);
603+
context.CreateInstances(2, disableObjects: true, disablePubSub: true, enableAOF: true, clusterCreds: clusterCreds, useAcl: true, FastAofTruncate: true, CommitFrequencyMs: -1, asyncReplay: false);
604+
var primaryEndpoint = (IPEndPoint)context.endpoints.First();
605+
var replicaEndpoint = (IPEndPoint)context.endpoints.Last();
606+
607+
ClassicAssert.AreNotEqual(primaryEndpoint, replicaEndpoint, "Should have different endpoints for nodes");
608+
609+
using var primaryConnection = ConnectionMultiplexer.Connect($"{primaryEndpoint.Address}:{primaryEndpoint.Port},user={UserName},password={Password}");
610+
var primaryServer = primaryConnection.GetServer(primaryEndpoint);
611+
612+
ClassicAssert.AreEqual("OK", (string)primaryServer.Execute("CLUSTER", ["ADDSLOTSRANGE", "0", "16383"], flags: CommandFlags.NoRedirect));
613+
ClassicAssert.AreEqual("OK", (string)primaryServer.Execute("CLUSTER", ["MEET", replicaEndpoint.Address.ToString(), replicaEndpoint.Port.ToString()], flags: CommandFlags.NoRedirect));
614+
615+
using var replicaConnection = ConnectionMultiplexer.Connect($"{replicaEndpoint.Address}:{replicaEndpoint.Port},user={UserName},password={Password}");
616+
var replicaServer = replicaConnection.GetServer(replicaEndpoint);
617+
618+
// Try to replicate from a server that doesn't exist
619+
var exc = Assert.Throws<RedisServerException>(() => replicaServer.Execute("CLUSTER", ["REPLICATE", Guid.NewGuid().ToString()], flags: CommandFlags.NoRedirect));
620+
ClassicAssert.IsTrue(exc.Message.StartsWith("ERR I don't know about node "));
621+
}
562622
}
563623
}

‎test/Garnet.test.cluster/ReplicationTests/ClusterReplicationAsyncReplay.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Garnet.test.cluster
77
{
88
[NonParallelizable]
9+
[Ignore("Skip to reduce CI duration.")]
910
public class ClusterReplicationAsyncReplay : ClusterReplicationBaseTests
1011
{
1112
[SetUp]

0 commit comments

Comments
 (0)