Skip to content

Commit 46fd29a

Browse files
author
Unmesh Joshi
committed
Refactor Paxos tests for educational clarity - break down complex scenarios into teachable phases with clear helper methods and workshop-ready structure
1 parent ee78cb3 commit 46fd29a

File tree

3 files changed

+815
-340
lines changed

3 files changed

+815
-340
lines changed

‎src/test/java/replicate/multipaxos/MultiPaxosTest.java‎

Lines changed: 262 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -19,116 +19,285 @@
1919

2020
import static org.junit.Assert.*;
2121

22+
/**
23+
* Educational tests for Multi-Paxos demonstrating distributed log consensus:
24+
* 1. Leader election and single value consensus
25+
* 2. Multiple values in replicated log
26+
* 3. Leader failure and recovery scenarios
27+
* 4. Log completion after partial failures
28+
*/
2229
public class MultiPaxosTest extends ClusterTest<MultiPaxos> {
30+
31+
// Educational naming: Greek city-states for MultiPaxos cluster
32+
private MultiPaxos athens;
33+
private MultiPaxos byzantium;
34+
private MultiPaxos cyrene;
35+
2336
@Before
24-
public void setUp() throws IOException {
25-
super.nodes = TestUtils.startCluster(nodeNames("athens", "byzantium", "cyrene"),
26-
(name, config, clock, clientConnectionAddress, peerConnectionAddress, peers) -> new MultiPaxos(name, clock, config, clientConnectionAddress, peerConnectionAddress, peers));
27-
28-
}
29-
30-
private static List<String> nodeNames(String... names) {
31-
return Arrays.asList(names);
37+
public void setupMultiPaxosCluster() throws IOException {
38+
super.nodes = TestUtils.startCluster(
39+
cityStateNames("athens", "byzantium", "cyrene"),
40+
(name, config, clock, clientConnectionAddress, peerConnectionAddress, peers) ->
41+
new MultiPaxos(name, clock, config, clientConnectionAddress, peerConnectionAddress, peers)
42+
);
43+
44+
// Cache references for educational clarity
45+
athens = nodes.get("athens");
46+
byzantium = nodes.get("byzantium");
47+
cyrene = nodes.get("cyrene");
3248
}
3349

50+
// ============================================================================
51+
// SCENARIO 1: Basic Leader Election and Single Value Consensus
52+
// ============================================================================
53+
3454
@Test
35-
public void setsSingleValue() throws Exception {
36-
var athens = nodes.get("athens");
37-
athens.leaderElection();
38-
TestUtils.waitUntilTrue(() -> {
39-
return athens.isLeader();
40-
}, "Waiting for leader election", Duration.ofSeconds(2));
41-
42-
var networkClient = new NetworkClient();
43-
byte[] command = new SetValueCommand("title", "Microservices").serialize();
44-
var setValueResponse = networkClient.sendAndReceive(new ExecuteCommandRequest(command), nodes.get("athens").getClientConnectionAddress(), ExecuteCommandResponse.class).getResult();
45-
assertEquals(Optional.of("Microservices"), setValueResponse.getResponse());
55+
public void demonstratesLeaderElectionAndBasicConsensus() throws Exception {
56+
// GIVEN: A MultiPaxos cluster with no leader
57+
// WHEN: Athens runs leader election
58+
electLeader(athens, "Athens should become leader");
59+
60+
// THEN: Athens can accept and replicate a single value
61+
String result = executeCommand("title", "Microservices", athens);
62+
assertEquals("Basic consensus through elected leader should work",
63+
"Microservices", result);
4664
}
4765

4866
@Test
49-
public void singleValueNullPaxosGetTest() throws Exception {
50-
var athens = nodes.get("athens");
51-
52-
athens.leaderElection();
53-
TestUtils.waitUntilTrue(() -> {
54-
return athens.isLeader();
55-
}, "Waiting for leader election", Duration.ofSeconds(2));
56-
57-
var networkClient = new NetworkClient();
58-
var getValueResponse = networkClient.sendAndReceive(new GetValueRequest("title"), nodes.get("athens").getClientConnectionAddress(), GetValueResponse.class).getResult();
59-
assertEquals(Optional.empty(), getValueResponse.value);
67+
public void handlesEmptyLogQueries() throws Exception {
68+
// GIVEN: An elected leader with empty log
69+
electLeader(athens, "Need leader for queries");
70+
71+
// WHEN: Client queries non-existent key
72+
// THEN: Should return empty, not error
73+
String result = queryValue("title", athens);
74+
assertNull("Empty log should return null for non-existent keys", result);
6075
}
6176

77+
// ============================================================================
78+
// SCENARIO 2: Multiple Commands in Replicated Log
79+
// ============================================================================
80+
6281
@Test
63-
public void singleValuePaxosGetTest() throws Exception {
64-
var athens = nodes.get("athens");
65-
66-
athens.leaderElection();
67-
TestUtils.waitUntilTrue(() -> {
68-
return athens.isLeader();
69-
}, "Waiting for leader election", Duration.ofSeconds(2));
70-
71-
var networkClient = new NetworkClient();
72-
byte[] command = new SetValueCommand("title", "Microservices").serialize();
73-
var setValueResponse = networkClient.sendAndReceive(new ExecuteCommandRequest(command), nodes.get("athens").getClientConnectionAddress(), ExecuteCommandResponse.class);
74-
75-
var getValueResponse = networkClient.sendAndReceive(new GetValueRequest("title"), nodes.get("athens").getClientConnectionAddress(), GetValueResponse.class).getResult();
76-
assertEquals(Optional.of("Microservices"), getValueResponse.value);
82+
public void demonstratesSequentialCommandReplication() throws Exception {
83+
// GIVEN: Athens is the elected leader
84+
electLeader(athens, "Athens leads the cluster");
85+
86+
// WHEN: Multiple commands are executed in sequence
87+
executeCommand("title", "Microservices", athens);
88+
String result = queryValue("title", athens);
89+
assertEquals("First command should be replicated", "Microservices", result);
90+
91+
// THEN: Later queries should return the stored value
92+
String laterResult = queryValue("title", athens);
93+
assertEquals("Value should persist in replicated log", "Microservices", laterResult);
7794
}
7895

79-
80-
@Test
81-
public void leaderElectionCompletesIncompletePaxosRuns() throws Exception {
82-
MultiPaxos athens = nodes.get("athens");
83-
MultiPaxos byzantium = nodes.get("byzantium");
84-
MultiPaxos cyrene = nodes.get("cyrene");
85-
86-
athens.leaderElection();
87-
88-
TestUtils.waitUntilTrue(() -> {
89-
return athens.isLeader();
90-
}, "Waiting for leader election", Duration.ofSeconds(2));
91-
96+
// ============================================================================
97+
// SCENARIO 3: Leader Failure and Log Recovery (The Key Educational Scenario)
98+
// This demonstrates the critical Multi-Paxos recovery mechanism
99+
// ============================================================================
100+
101+
@Test
102+
public void demonstratesLeaderFailureRecoveryAndLogCompletion() throws Exception {
103+
// PHASE 1: Establish Athens as leader and execute first command successfully
104+
LeadershipPhase phase1 = establishInitialLeadership();
105+
106+
// PHASE 2: Simulate network partition during second command
107+
PartialFailureScenario phase2 = simulatePartialCommandFailure();
108+
109+
// PHASE 3: New leader election and automatic log completion
110+
LogRecoveryResult phase3 = demonstrateAutomaticLogRecovery();
111+
112+
// FINAL VERIFICATION: All nodes have consistent, complete log
113+
verifyLogConsistencyAcrossCluster();
114+
}
115+
116+
// ============================================================================
117+
// Educational Helper Methods - Breaking down complex Multi-Paxos scenarios
118+
// ============================================================================
119+
120+
private LeadershipPhase establishInitialLeadership() throws Exception {
121+
// EDUCATIONAL STEP 1: Athens becomes leader
122+
electLeader(athens, "Athens establishes initial leadership");
123+
124+
// EDUCATIONAL STEP 2: Successfully replicate first command
125+
String firstResult = executeCommand("title", "Microservices", athens);
126+
assertEquals("First command should succeed under stable leadership",
127+
"Microservices", firstResult);
128+
129+
// EDUCATIONAL VERIFICATION: All nodes have the first entry
130+
verifyLogSizeAcrossCluster(1, "After first successful command");
131+
verifyAllNodesHaveCommittedEntry(0, "First entry should be committed everywhere");
132+
133+
return new LeadershipPhase("Athens", "Microservices", true);
134+
}
135+
136+
private PartialFailureScenario simulatePartialCommandFailure() throws Exception {
137+
// EDUCATIONAL SETUP: Create network partition that prevents full replication
138+
athens.dropMessagesTo(byzantium); // Athens can't reach Byzantium
139+
athens.dropMessagesTo(cyrene); // Athens can't reach Cyrene
140+
141+
// EDUCATIONAL ATTEMPT: Try to execute second command (should fail)
92142
var networkClient = new NetworkClient();
93-
byte[] command = new SetValueCommand("title", "Microservices").serialize();
94-
var setValueResponse = networkClient.sendAndReceive(new ExecuteCommandRequest(command), athens.getClientConnectionAddress(), ExecuteCommandResponse.class);
95-
96-
athens.dropMessagesTo(byzantium); //propose messages fail
97-
athens.dropMessagesTo(cyrene); //propose messages fail
98-
99-
100-
command = new SetValueCommand("author", "Martin").serialize();
101-
var response1 = networkClient.sendAndReceive(new ExecuteCommandRequest(command), athens.getClientConnectionAddress(), ExecuteCommandResponse.class);
102-
assertTrue("Expected to fail because athens will be unable to reach quorum", response1.isError());
103-
104-
assertEquals(2, athens.paxosLog.size()); //uncommitted second entry
105-
assertEquals(1, byzantium.paxosLog.size()); //only first entry.
106-
assertEquals(1, cyrene.paxosLog.size()); //only first entry.
107-
108-
assertTrue(athens.paxosLog.get(0).committedValue().isPresent());
109-
assertTrue(byzantium.paxosLog.get(0).committedValue().isPresent());
110-
assertTrue(cyrene.paxosLog.get(0).committedValue().isPresent());
111-
112-
assertFalse(athens.paxosLog.get(1).committedValue().isPresent());
113-
143+
byte[] command = new SetValueCommand("author", "Martin").serialize();
144+
var response = networkClient.sendAndReceive(
145+
new ExecuteCommandRequest(command),
146+
athens.getClientConnectionAddress(),
147+
ExecuteCommandResponse.class
148+
);
149+
150+
// EDUCATIONAL ASSERTION: Command should fail due to inability to reach quorum
151+
assertTrue("Command should fail - Athens cannot reach quorum due to network partition",
152+
response.isError());
153+
154+
// EDUCATIONAL STATE VERIFICATION: Check log states after partial failure
155+
assertEquals("Athens should have 2 entries (1 committed, 1 pending)",
156+
2, athens.paxosLog.size());
157+
assertEquals("Byzantium should only have 1 entry (the committed one)",
158+
1, byzantium.paxosLog.size());
159+
assertEquals("Cyrene should only have 1 entry (the committed one)",
160+
1, cyrene.paxosLog.size());
161+
162+
// EDUCATIONAL KEY INSIGHT: First entry is committed everywhere, second is only on Athens
163+
verifyAllNodesHaveCommittedEntry(0, "First entry remains committed despite network issues");
164+
assertFalse("Athens' second entry should be uncommitted due to network failure",
165+
athens.paxosLog.get(1).committedValue().isPresent());
166+
167+
// EDUCATIONAL VERIFICATION: The failed command value is not accessible
168+
assertNull("Failed command should not be queryable", athens.getValue("author"));
169+
170+
return new PartialFailureScenario("author", "Martin", false);
171+
}
172+
173+
private LogRecoveryResult demonstrateAutomaticLogRecovery() throws Exception {
174+
// EDUCATIONAL SETUP: Network heals, Byzantium becomes new leader
114175
athens.reconnectTo(byzantium);
115176
athens.reconnectTo(cyrene);
177+
178+
// EDUCATIONAL KEY POINT: New leader election triggers log completion
179+
electLeader(byzantium, "Byzantium becomes new leader and will complete incomplete log entries");
180+
181+
// EDUCATIONAL VERIFICATION: Log completion should happen automatically
182+
verifyLogSizeAcrossCluster(2, "New leader should complete all pending log entries");
183+
184+
// EDUCATIONAL KEY INSIGHT: The previously failed command is now successful
185+
assertEquals("Previously failed command should now be accessible after log completion",
186+
"Martin", athens.getValue("author"));
187+
assertEquals("All nodes should have the recovered value",
188+
"Martin", byzantium.getValue("author"));
189+
assertEquals("Cyrene should also have the recovered value",
190+
"Martin", cyrene.getValue("author"));
191+
192+
return new LogRecoveryResult("Martin", 2, true);
193+
}
194+
195+
private void verifyLogConsistencyAcrossCluster() {
196+
// EDUCATIONAL FINAL CHECK: All nodes should have identical logs
197+
assertEquals("All nodes should have same log size after recovery",
198+
athens.paxosLog.size(), byzantium.paxosLog.size());
199+
assertEquals("Athens and Cyrene should have same log size",
200+
athens.paxosLog.size(), cyrene.paxosLog.size());
201+
202+
// EDUCATIONAL VERIFICATION: All values should be accessible from any node
203+
assertEquals("Title should be accessible from any node",
204+
"Microservices", athens.getValue("title"));
205+
assertEquals("Author should be accessible from any node",
206+
"Martin", byzantium.getValue("author"));
207+
}
116208

117-
assertNull(athens.getValue("author"));
118-
119-
//election which is equivalent to prepare phase of basic paxos, checks
120-
//and completes pending log entries from majority quorum of the servers.
121-
byzantium.leaderElection();
122-
TestUtils.waitUntilTrue(() -> {
123-
return byzantium.isLeader();
124-
}, "Waiting for leader election", Duration.ofSeconds(2));
125-
126-
assertEquals(2, athens.paxosLog.size());
127-
assertEquals(2, byzantium.paxosLog.size());
128-
assertEquals(2, cyrene.paxosLog.size());
209+
// ============================================================================
210+
// Educational Utility Methods - Making tests more readable and teachable
211+
// ============================================================================
212+
213+
private void electLeader(MultiPaxos node, String educationalContext) throws Exception {
214+
node.leaderElection();
215+
TestUtils.waitUntilTrue(() -> node.isLeader(),
216+
"Waiting for leader election: " + educationalContext,
217+
Duration.ofSeconds(2));
218+
assertTrue(educationalContext + " - Node should be leader after election",
219+
node.isLeader());
220+
}
221+
222+
private String executeCommand(String key, String value, MultiPaxos leader) throws Exception {
223+
var networkClient = new NetworkClient();
224+
byte[] command = new SetValueCommand(key, value).serialize();
225+
var response = networkClient.sendAndReceive(
226+
new ExecuteCommandRequest(command),
227+
leader.getClientConnectionAddress(),
228+
ExecuteCommandResponse.class
229+
).getResult();
230+
231+
return response.getResponse().orElse(null);
232+
}
233+
234+
private String queryValue(String key, MultiPaxos node) throws Exception {
235+
var networkClient = new NetworkClient();
236+
var response = networkClient.sendAndReceive(
237+
new GetValueRequest(key),
238+
node.getClientConnectionAddress(),
239+
GetValueResponse.class
240+
).getResult();
241+
242+
return response.value.orElse(null);
243+
}
244+
245+
private void verifyLogSizeAcrossCluster(int expectedSize, String context) {
246+
assertEquals(context + " - Athens log size", expectedSize, athens.paxosLog.size());
247+
assertEquals(context + " - Byzantium log size", expectedSize, byzantium.paxosLog.size());
248+
assertEquals(context + " - Cyrene log size", expectedSize, cyrene.paxosLog.size());
249+
}
250+
251+
private void verifyAllNodesHaveCommittedEntry(int index, String context) {
252+
assertTrue(context + " - Athens entry " + index + " should be committed",
253+
athens.paxosLog.get(index).committedValue().isPresent());
254+
assertTrue(context + " - Byzantium entry " + index + " should be committed",
255+
byzantium.paxosLog.get(index).committedValue().isPresent());
256+
assertTrue(context + " - Cyrene entry " + index + " should be committed",
257+
cyrene.paxosLog.get(index).committedValue().isPresent());
258+
}
129259

130-
assertEquals("Martin", athens.getValue("author"));
131-
assertEquals("Martin", byzantium.getValue("author"));
132-
assertEquals("Martin", cyrene.getValue("author"));
260+
// ============================================================================
261+
// Educational Data Classes - Making test results more structured and readable
262+
// ============================================================================
263+
264+
private static class LeadershipPhase {
265+
final String leader;
266+
final String firstValue;
267+
final boolean successful;
268+
269+
LeadershipPhase(String leader, String firstValue, boolean successful) {
270+
this.leader = leader;
271+
this.firstValue = firstValue;
272+
this.successful = successful;
273+
}
274+
}
275+
276+
private static class PartialFailureScenario {
277+
final String attemptedKey;
278+
final String attemptedValue;
279+
final boolean succeeded;
280+
281+
PartialFailureScenario(String attemptedKey, String attemptedValue, boolean succeeded) {
282+
this.attemptedKey = attemptedKey;
283+
this.attemptedValue = attemptedValue;
284+
this.succeeded = succeeded;
285+
}
286+
}
287+
288+
private static class LogRecoveryResult {
289+
final String recoveredValue;
290+
final int finalLogSize;
291+
final boolean recoverySuccessful;
292+
293+
LogRecoveryResult(String recoveredValue, int finalLogSize, boolean recoverySuccessful) {
294+
this.recoveredValue = recoveredValue;
295+
this.finalLogSize = finalLogSize;
296+
this.recoverySuccessful = recoverySuccessful;
297+
}
298+
}
299+
300+
private static List<String> cityStateNames(String... names) {
301+
return Arrays.asList(names);
133302
}
134303
}

0 commit comments

Comments
 (0)