Skip to content

Commit 38c37ca

Browse files
poorbarcodepoorbarcode
committed
test CI
1 parent 9c41661 commit 38c37ca

File tree

1 file changed

+255
-90
lines changed

1 file changed

+255
-90
lines changed

‎bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java‎

Lines changed: 255 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.junit.Assert.assertEquals;
2323
import static org.junit.Assert.assertNotNull;
2424
import static org.junit.Assert.assertNull;
25+
import static org.junit.Assert.assertTrue;
2526

2627
import java.io.IOException;
2728
import java.net.URI;
@@ -30,10 +31,13 @@
3031
import java.util.Optional;
3132
import java.util.SortedMap;
3233
import java.util.concurrent.CountDownLatch;
34+
import java.util.concurrent.TimeUnit;
3335
import org.apache.bookkeeper.client.BKException;
3436
import org.apache.bookkeeper.client.BookKeeper.DigestType;
37+
import org.apache.bookkeeper.client.BookKeeperTestClient;
3538
import org.apache.bookkeeper.client.LedgerHandle;
3639
import org.apache.bookkeeper.common.util.OrderedScheduler;
40+
import org.apache.bookkeeper.conf.ServerConfiguration;
3741
import org.apache.bookkeeper.meta.LedgerManager;
3842
import org.apache.bookkeeper.meta.LedgerManagerFactory;
3943
import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
@@ -136,96 +140,6 @@ public void tearDown() throws Exception {
136140
}
137141
}
138142

139-
// @Test
140-
// public void testOpenLedgers1() throws Exception {
141-
// testOpenLedgers();
142-
// }
143-
//
144-
// @Test
145-
// public void testClosedLedgers1() throws Exception {
146-
// testOpenLedgers();
147-
// }
148-
//
149-
// @Test
150-
// public void testOpenLedgers2() throws Exception {
151-
// testOpenLedgers();
152-
// }
153-
//
154-
// @Test
155-
// public void testClosedLedgers2() throws Exception {
156-
// testOpenLedgers();
157-
// }
158-
//
159-
// @Test
160-
// public void testOpenLedgers3() throws Exception {
161-
// testOpenLedgers();
162-
// }
163-
//
164-
// @Test
165-
// public void testClosedLedgers3() throws Exception {
166-
// testOpenLedgers();
167-
// }
168-
//
169-
// @Test
170-
// public void testOpenLedgers4() throws Exception {
171-
// testOpenLedgers();
172-
// }
173-
//
174-
// @Test
175-
// public void testClosedLedgers4() throws Exception {
176-
// testOpenLedgers();
177-
// }
178-
//
179-
// @Test
180-
// public void testOpenLedgers5() throws Exception {
181-
// testOpenLedgers();
182-
// }
183-
//
184-
// @Test
185-
// public void testClosedLedgers5() throws Exception {
186-
// testOpenLedgers();
187-
// }
188-
//
189-
// @Test
190-
// public void testOpenLedgers6() throws Exception {
191-
// testOpenLedgers();
192-
// }
193-
//
194-
// @Test
195-
// public void testClosedLedgers6() throws Exception {
196-
// testOpenLedgers();
197-
// }
198-
//
199-
// @Test
200-
// public void testOpenLedgers7() throws Exception {
201-
// testOpenLedgers();
202-
// }
203-
//
204-
// @Test
205-
// public void testClosedLedgers7() throws Exception {
206-
// testOpenLedgers();
207-
// }
208-
//
209-
// @Test
210-
// public void testOpenLedgers8() throws Exception {
211-
// testOpenLedgers();
212-
// }
213-
//
214-
// @Test
215-
// public void testClosedLedgers8() throws Exception {
216-
// testOpenLedgers();
217-
// }
218-
//
219-
// @Test
220-
// public void testOpenLedgers9() throws Exception {
221-
// testOpenLedgers();
222-
// }
223-
//
224-
// @Test
225-
// public void testClosedLedgers9() throws Exception {
226-
// testOpenLedgers();
227-
// }
228-
229143
/**
230144
* Test verifies publish urLedger by Auditor and replication worker is
231145
* picking up the entries and finishing the rereplication of open ledger.
@@ -357,6 +271,257 @@ public void testClosedLedgers() throws Exception {
357271
LOG.info("===> Finished test close ledgers");
358272
}
359273

274+
/**
275+
* Verify the published urledgers of deleted ledgers(those ledgers where
276+
* deleted after publishing as urledgers by Auditor) should be cleared off
277+
* by the newly selected replica bookie.
278+
*/
279+
@Test
280+
public void testNoSuchLedgerExists() throws Exception {
281+
LOG.info("===> Starting testNoSuchLedgerExists");
282+
List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(2, 5);
283+
CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size());
284+
for (LedgerHandle lh : listOfLedgerHandle) {
285+
assertNull("UrLedger already exists!",
286+
watchUrLedgerNode(getUrLedgerZNode(lh), latch));
287+
}
288+
BookieId replicaToKillAddr = listOfLedgerHandle.get(0)
289+
.getLedgerMetadata().getAllEnsembles()
290+
.get(0L).get(0);
291+
killBookie(replicaToKillAddr);
292+
replicaToKillAddr = listOfLedgerHandle.get(0)
293+
.getLedgerMetadata().getAllEnsembles()
294+
.get(0L).get(0);
295+
killBookie(replicaToKillAddr);
296+
// waiting to publish urLedger znode by Auditor
297+
latch.await();
298+
299+
latch = new CountDownLatch(listOfLedgerHandle.size());
300+
for (LedgerHandle lh : listOfLedgerHandle) {
301+
assertNotNull("UrLedger doesn't exists!",
302+
watchUrLedgerNode(getUrLedgerZNode(lh), latch));
303+
}
304+
305+
// delete ledgers
306+
for (LedgerHandle lh : listOfLedgerHandle) {
307+
bkc.deleteLedger(lh.getId());
308+
}
309+
startNewBookie();
310+
311+
// waiting to delete published urledgers, since it doesn't exists
312+
latch.await();
313+
314+
for (LedgerHandle lh : listOfLedgerHandle) {
315+
assertNull("UrLedger still exists after rereplication",
316+
watchUrLedgerNode(getUrLedgerZNode(lh), latch));
317+
}
318+
LOG.info("===> Finished testNoSuchLedgerExists");
319+
}
320+
321+
/**
322+
* Test that if an empty ledger loses the bookie not in the quorum for entry 0, it will
323+
* still be openable when it loses enough bookies to lose a whole quorum.
324+
*/
325+
@Test
326+
public void testEmptyLedgerLosesQuorumEventually() throws Exception {
327+
LOG.info("===> Starting testEmptyLedgerLosesQuorumEventually");
328+
LedgerHandle lh = bkc.createLedger(3, 2, 2, DigestType.CRC32, PASSWD);
329+
CountDownLatch latch = new CountDownLatch(1);
330+
String urZNode = getUrLedgerZNode(lh);
331+
watchUrLedgerNode(urZNode, latch);
332+
333+
BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(2);
334+
LOG.info("Killing last bookie, {}, in ensemble {}", replicaToKill,
335+
lh.getLedgerMetadata().getAllEnsembles().get(0L));
336+
killBookie(replicaToKill);
337+
startNewBookie();
338+
339+
getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
340+
341+
assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS));
342+
latch = new CountDownLatch(1);
343+
Stat s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated
344+
if (s != null) {
345+
assertTrue("Should be marked as replicated", latch.await(15, TimeUnit.SECONDS));
346+
}
347+
348+
replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(1);
349+
LOG.info("Killing second bookie, {}, in ensemble {}", replicaToKill,
350+
lh.getLedgerMetadata().getAllEnsembles().get(0L));
351+
killBookie(replicaToKill);
352+
353+
getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
354+
355+
assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS));
356+
latch = new CountDownLatch(1);
357+
s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated
358+
359+
startNewBookie();
360+
getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs
361+
362+
if (s != null) {
363+
assertTrue("Should be marked as replicated", latch.await(20, TimeUnit.SECONDS));
364+
}
365+
366+
// should be able to open ledger without issue
367+
bkc.openLedger(lh.getId(), DigestType.CRC32, PASSWD);
368+
LOG.info("===> Finished testEmptyLedgerLosesQuorumEventually");
369+
}
370+
371+
/**
372+
* Test verifies bookie recovery, the host (recorded via ipaddress in
373+
* ledgermetadata).
374+
*/
375+
@Test
376+
public void testLedgerMetadataContainsIpAddressAsBookieID()
377+
throws Exception {
378+
LOG.info("===> Starting testLedgerMetadataContainsIpAddressAsBookieID");
379+
stopBKCluster();
380+
bkc = new BookKeeperTestClient(baseClientConf);
381+
// start bookie with useHostNameAsBookieID=false, as old bookie
382+
ServerConfiguration serverConf1 = newServerConfiguration();
383+
// start 2 more bookies with useHostNameAsBookieID=true
384+
ServerConfiguration serverConf2 = newServerConfiguration();
385+
serverConf2.setUseHostNameAsBookieID(true);
386+
ServerConfiguration serverConf3 = newServerConfiguration();
387+
serverConf3.setUseHostNameAsBookieID(true);
388+
startAndAddBookie(serverConf1);
389+
startAndAddBookie(serverConf2);
390+
startAndAddBookie(serverConf3);
391+
392+
List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
393+
LedgerHandle lh = listOfLedgerHandle.get(0);
394+
int ledgerReplicaIndex = 0;
395+
final SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();
396+
final List<BookieId> bkAddresses = ensembles.get(0L);
397+
BookieId replicaToKillAddr = bkAddresses.get(0);
398+
for (BookieId bookieSocketAddress : bkAddresses) {
399+
if (!isCreatedFromIp(bookieSocketAddress)) {
400+
replicaToKillAddr = bookieSocketAddress;
401+
LOG.info("Kill bookie which has registered using hostname");
402+
break;
403+
}
404+
}
405+
406+
final String urLedgerZNode = getUrLedgerZNode(lh);
407+
ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
408+
409+
CountDownLatch latch = new CountDownLatch(1);
410+
assertNull("UrLedger already exists!",
411+
watchUrLedgerNode(urLedgerZNode, latch));
412+
413+
LOG.info("Killing Bookie :" + replicaToKillAddr);
414+
killBookie(replicaToKillAddr);
415+
416+
// waiting to publish urLedger znode by Auditor
417+
latch.await();
418+
latch = new CountDownLatch(1);
419+
LOG.info("Watching on urLedgerPath:" + urLedgerZNode
420+
+ " to know the status of rereplication process");
421+
assertNotNull("UrLedger doesn't exists!",
422+
watchUrLedgerNode(urLedgerZNode, latch));
423+
424+
// starting the replication service, so that he will be able to act as
425+
// target bookie
426+
ServerConfiguration serverConf = newServerConfiguration();
427+
serverConf.setUseHostNameAsBookieID(false);
428+
startAndAddBookie(serverConf);
429+
430+
int newBookieIndex = lastBookieIndex();
431+
BookieServer newBookieServer = serverByIndex(newBookieIndex);
432+
433+
if (LOG.isDebugEnabled()) {
434+
LOG.debug("Waiting to finish the replication of failed bookie : "
435+
+ replicaToKillAddr);
436+
}
437+
latch.await();
438+
439+
// grace period to update the urledger metadata in zookeeper
440+
LOG.info("Waiting to update the urledger metadata in zookeeper");
441+
442+
verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
443+
listOfLedgerHandle.get(0), ledgerReplicaIndex);
444+
LOG.info("===> Finished testLedgerMetadataContainsIpAddressAsBookieID");
445+
}
446+
447+
/**
448+
* Test verifies bookie recovery, the host (recorded via useHostName in
449+
* ledgermetadata).
450+
*/
451+
@Test
452+
public void testLedgerMetadataContainsHostNameAsBookieID()
453+
throws Exception {
454+
LOG.info("===> Starting testLedgerMetadataContainsHostNameAsBookieID");
455+
stopBKCluster();
456+
457+
bkc = new BookKeeperTestClient(baseClientConf);
458+
// start bookie with useHostNameAsBookieID=false, as old bookie
459+
ServerConfiguration serverConf1 = newServerConfiguration();
460+
// start 2 more bookies with useHostNameAsBookieID=true
461+
ServerConfiguration serverConf2 = newServerConfiguration();
462+
serverConf2.setUseHostNameAsBookieID(true);
463+
ServerConfiguration serverConf3 = newServerConfiguration();
464+
serverConf3.setUseHostNameAsBookieID(true);
465+
startAndAddBookie(serverConf1);
466+
startAndAddBookie(serverConf2);
467+
startAndAddBookie(serverConf3);
468+
469+
List<LedgerHandle> listOfLedgerHandle = createLedgersAndAddEntries(1, 5);
470+
LedgerHandle lh = listOfLedgerHandle.get(0);
471+
int ledgerReplicaIndex = 0;
472+
final SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();
473+
final List<BookieId> bkAddresses = ensembles.get(0L);
474+
BookieId replicaToKillAddr = bkAddresses.get(0);
475+
for (BookieId bookieSocketAddress : bkAddresses) {
476+
if (isCreatedFromIp(bookieSocketAddress)) {
477+
replicaToKillAddr = bookieSocketAddress;
478+
LOG.info("Kill bookie which has registered using ipaddress");
479+
break;
480+
}
481+
}
482+
483+
final String urLedgerZNode = getUrLedgerZNode(lh);
484+
ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr);
485+
486+
CountDownLatch latch = new CountDownLatch(1);
487+
assertNull("UrLedger already exists!",
488+
watchUrLedgerNode(urLedgerZNode, latch));
489+
490+
LOG.info("Killing Bookie :" + replicaToKillAddr);
491+
killBookie(replicaToKillAddr);
492+
493+
// waiting to publish urLedger znode by Auditor
494+
latch.await();
495+
latch = new CountDownLatch(1);
496+
LOG.info("Watching on urLedgerPath:" + urLedgerZNode
497+
+ " to know the status of rereplication process");
498+
assertNotNull("UrLedger doesn't exists!",
499+
watchUrLedgerNode(urLedgerZNode, latch));
500+
501+
// creates new bkclient
502+
bkc = new BookKeeperTestClient(baseClientConf);
503+
// starting the replication service, so that he will be able to act as
504+
// target bookie
505+
ServerConfiguration serverConf = newServerConfiguration();
506+
serverConf.setUseHostNameAsBookieID(true);
507+
startAndAddBookie(serverConf);
508+
509+
int newBookieIndex = lastBookieIndex();
510+
BookieServer newBookieServer = serverByIndex(newBookieIndex);
511+
512+
if (LOG.isDebugEnabled()) {
513+
LOG.debug("Waiting to finish the replication of failed bookie : "
514+
+ replicaToKillAddr);
515+
}
516+
latch.await();
517+
518+
// grace period to update the urledger metadata in zookeeper
519+
LOG.info("Waiting to update the urledger metadata in zookeeper");
520+
521+
verifyLedgerEnsembleMetadataAfterReplication(newBookieServer,
522+
listOfLedgerHandle.get(0), ledgerReplicaIndex);
523+
LOG.info("===> Finished testLedgerMetadataContainsHostNameAsBookieID");
524+
}
360525

361526
private int getReplicaIndexInLedger(LedgerHandle lh, BookieId replicaToKill) {
362527
SortedMap<Long, ? extends List<BookieId>> ensembles = lh.getLedgerMetadata().getAllEnsembles();

0 commit comments

Comments
 (0)