Skip to content

Commit 07de650

Browse files
ethqunzhongethqunzhong
authored
remove in address2Region while bookie left to get correct rack info (#4504)
1 parent 7855518 commit 07de650

File tree

2 files changed

+136
-0
lines changed

2 files changed

+136
-0
lines changed

‎bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,23 @@ protected String parseBookieRegion(BookieId addr) {
117117

118118
@Override
119119
public void handleBookiesThatLeft(Set<BookieId> leftBookies) {
120+
// case 1: (In some situation, eg, Broker and bookie restart concurrently.)
121+
//1. Bookie X join cluster for the first time, encounters a region exception, and `address2Region` record X's
122+
// region as default-region.
123+
//2. Bookie X left cluster and is removed from knownBookies, but address2Region retains the information of
124+
// bookie X.
125+
//3. update Bookie X's rack info, and calling `onBookieRackChange` will only update address2Region for
126+
// addresses present in knownBookies; therefore, bookie X's region info is not updated.
127+
//4. Bookie X join cluster again, since address2Region contains the previous default-region information,
128+
// getRegion will directly use cached data, resulting of an incorrect region.
129+
130+
// The bookie region is initialized to "default-region" in address2Region.
131+
// We should ensure that when a bookie leaves the cluster,
132+
// we also clean up the corresponding region information for that bookie in address2Region,
133+
// so that it can update the correct region for the bookie during onBookieRackChange and
134+
// handleBookiesThatJoined.
135+
// to avoid traffic skew in ensemble selection.
136+
leftBookies.forEach(address2Region::remove);
120137
super.handleBookiesThatLeft(leftBookies);
121138

122139
for (TopologyAwareEnsemblePlacementPolicy policy: perRegionPlacement.values()) {

‎bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL;
3030
import static org.junit.jupiter.api.Assertions.assertEquals;
3131
import static org.junit.jupiter.api.Assertions.assertNotEquals;
32+
import static org.junit.jupiter.api.Assertions.assertNotNull;
33+
import static org.junit.jupiter.api.Assertions.assertNull;
3234
import static org.junit.jupiter.api.Assertions.assertTrue;
3335
import static org.junit.jupiter.api.Assertions.fail;
3436
import static org.mockito.Mockito.mock;
@@ -2149,4 +2151,121 @@ public void testNewEnsemblePickLocalRegionBookies()
21492151
LOG.info("Bookie1 Count: {}, Bookie8 Count: {}, Bookie9 Count: {}", bookie1Count, bookie8Count, bookie9Count);
21502152

21512153
}
2154+
2155+
@Test
2156+
public void testBookieLeftThenJoinWithDNSResolveFailed() throws Exception {
2157+
2158+
BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181);
2159+
BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.2", 3181);
2160+
BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.3", 3181);
2161+
BookieSocketAddress addr4 = new BookieSocketAddress("127.0.1.4", 3181);
2162+
2163+
// init dns mapping
2164+
// 2. mock dns resolver failed, use default region and rack.
2165+
// addr1 rack info. /region-1/default-rack -> /default-region/default-rack.
2166+
2167+
// 1. mock addr1 dns resolver failed and use default region and rack.
2168+
StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/default-region/default-rack");
2169+
StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region-1/default-rack");
2170+
StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region-2/default-rack");
2171+
StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region-3/default-rack");
2172+
2173+
// init cluster
2174+
Set<BookieId> addrs = Sets.newHashSet(addr1.toBookieId(),
2175+
addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId());
2176+
repp.onClusterChanged(addrs, new HashSet<>());
2177+
2178+
assertEquals(4, repp.knownBookies.size());
2179+
assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
2180+
assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
2181+
assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
2182+
assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
2183+
2184+
assertEquals(4, repp.perRegionPlacement.size());
2185+
TopologyAwareEnsemblePlacementPolicy unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion");
2186+
assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size());
2187+
assertEquals("/default-region/default-rack",
2188+
unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
2189+
2190+
TopologyAwareEnsemblePlacementPolicy region1Placement = repp.perRegionPlacement.get("region-1");
2191+
assertEquals(1, region1Placement.knownBookies.keySet().size());
2192+
assertEquals("/region-1/default-rack",
2193+
region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
2194+
2195+
TopologyAwareEnsemblePlacementPolicy region2Placement = repp.perRegionPlacement.get("region-2");
2196+
assertEquals(1, region2Placement.knownBookies.keySet().size());
2197+
assertEquals("/region-2/default-rack",
2198+
region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
2199+
2200+
TopologyAwareEnsemblePlacementPolicy region3Placement = repp.perRegionPlacement.get("region-3");
2201+
assertEquals(1, region3Placement.knownBookies.keySet().size());
2202+
assertEquals("/region-3/default-rack",
2203+
region3Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
2204+
2205+
assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId()));
2206+
assertEquals("region-1", repp.address2Region.get(addr2.toBookieId()));
2207+
assertEquals("region-2", repp.address2Region.get(addr3.toBookieId()));
2208+
assertEquals("region-3", repp.address2Region.get(addr4.toBookieId()));
2209+
2210+
// 2. addr1 bookie shutdown and decommission
2211+
addrs.remove(addr1.toBookieId());
2212+
repp.onClusterChanged(addrs, new HashSet<>());
2213+
2214+
assertEquals(3, repp.knownBookies.size());
2215+
assertNull(repp.knownBookies.get(addr1.toBookieId()));
2216+
assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
2217+
assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
2218+
assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
2219+
2220+
// UnknownRegion,region-1,region-2,region-3
2221+
assertEquals(4, repp.perRegionPlacement.size());
2222+
// after addr1 bookie left, it should remove from locally address2Region
2223+
assertNull(repp.address2Region.get(addr1.toBookieId()));
2224+
assertEquals("region-1", repp.address2Region.get(addr2.toBookieId()));
2225+
assertEquals("region-2", repp.address2Region.get(addr3.toBookieId()));
2226+
assertEquals("region-3", repp.address2Region.get(addr4.toBookieId()));
2227+
2228+
2229+
// 3. addr1 bookie start and join
2230+
addrs.add(addr1.toBookieId());
2231+
repp.onClusterChanged(addrs, new HashSet<>());
2232+
2233+
assertEquals(4, repp.knownBookies.size());
2234+
assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
2235+
assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation());
2236+
assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation());
2237+
assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation());
2238+
2239+
// UnknownRegion,region-1,region-2,region-3
2240+
assertEquals(4, repp.perRegionPlacement.size());
2241+
assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId()));
2242+
// addr1 bookie belongs to unknown region
2243+
unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion");
2244+
assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size());
2245+
assertEquals("/default-region/default-rack",
2246+
unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
2247+
2248+
// 4. Update the correct rack.
2249+
// change addr1 rack info. /default-region/default-rack -> /region-1/default-rack.
2250+
List<BookieSocketAddress> bookieAddressList = new ArrayList<>();
2251+
List<String> rackList = new ArrayList<>();
2252+
bookieAddressList.add(addr1);
2253+
rackList.add("/region-1/default-rack");
2254+
// onBookieRackChange
2255+
StaticDNSResolver.changeRack(bookieAddressList, rackList);
2256+
2257+
assertEquals(4, repp.perRegionPlacement.size());
2258+
// addr1 bookie, oldRegion=default-region, newRegion=region-1
2259+
assertEquals("region-1", repp.address2Region.get(addr1.toBookieId()));
2260+
2261+
unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion");
2262+
assertEquals(0, unknownRegionPlacement.knownBookies.keySet().size());
2263+
assertNotNull(unknownRegionPlacement.historyBookies.get(addr1.toBookieId()));
2264+
2265+
2266+
region1Placement = repp.perRegionPlacement.get("region-1");
2267+
assertEquals(2, region1Placement.knownBookies.keySet().size());
2268+
assertEquals("/region-1/default-rack",
2269+
region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation());
2270+
}
21522271
}

0 commit comments

Comments
 (0)