Skip to content

support ipv6 address validation #3158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wangjialing218 wants to merge 6 commits into from
Closed

support ipv6 address validation #3158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a574ab1
fix validate hostname fail for ipv6 address
wangjialing218 Mar 30, 2022
5e45730
add test
wangjialing218 Mar 30, 2022
0ba902d
add ipv6 test case
wangjialing218 Aug 8, 2022
3ea39db
fix for code style
wangjialing218 Mar 20, 2023
c879e26
change URI.create to ServiceURI.create for IPV6 support
wangjialing218 Apr 10, 2023
3cba9c0
add ZOOKEEPER_SERVER_PRINCIPAL to avoid connect zookeeper slow
wangjialing218 Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 47 additions & 13 deletions bookkeeper-common/src/main/java/org/apache/bookkeeper/common/net/ServiceURI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.google.common.base.Strings;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.AccessLevel;
Expand Down Expand Up @@ -145,6 +146,39 @@ public class ServiceURI {
public static ServiceURI create(String uriStr) {
checkNotNull(uriStr, "service uri string is null");

if (uriStr.contains("[") && uriStr.contains("]")) {
// deal with ipv6 address
Splitter splitter = Splitter.on(CharMatcher.anyOf(",;"));
List<String> hosts = splitter.splitToList(uriStr);

if (hosts.size() > 1) {
// deal with multi ipv6 hosts
String firstHost = hosts.get(0);
String lastHost = hosts.get(hosts.size() - 1);
boolean hasPath = lastHost.contains("/");
String path = hasPath ? lastHost.substring(lastHost.indexOf("/")) : "";
firstHost += path;

URI uri = URI.create(firstHost);
ServiceURI serviceURI = create(uri);

List<String> multiHosts = new ArrayList<>();
multiHosts.add(serviceURI.getServiceHosts()[0]);
multiHosts.addAll(hosts.subList(1, hosts.size()));
multiHosts = multiHosts
.stream()
.map(host -> validateHostName(serviceURI.getServiceName(), host))
.collect(Collectors.toList());
return new ServiceURI(
serviceURI.getServiceName(),
serviceURI.getServiceInfos(),
serviceURI.getServiceUser(),
multiHosts.toArray(new String[multiHosts.size()]),
serviceURI.getServicePath(),
serviceURI.getUri());
}
Copy link
Contributor

@hangc0276 hangc0276 Jul 26, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to deal with hosts.size() == 1 ?

Copy link
Author

@wangjialing218 wangjialing218 Jul 27, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, URI.create(uriStr) could handle this case, testIPv6Address() cover this test case

}

// a service uri first should be a valid java.net.URI
URI uri = URI.create(uriStr);

Expand Down Expand Up @@ -215,21 +249,21 @@ public static ServiceURI create(URI uri) {
}

private static String validateHostName(String serviceName, String hostname) {
String[] parts = hostname.split(":");
if (parts.length >= 3) {
URI uri = null;
try {
uri = URI.create("dummyscheme://" + hostname);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Invalid hostname : " + hostname);
} else if (parts.length == 2) {
try {
Integer.parseUnsignedInt(parts[1]);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException("Invalid hostname : " + hostname);
}
return hostname;
} else if (parts.length == 1 && serviceName.toLowerCase().equals(SERVICE_BK)) {
return hostname + ":" + SERVICE_BK_PORT;
} else {
return hostname;
}
String host = uri.getHost();
if (host == null) {
throw new IllegalArgumentException("Invalid hostname : " + hostname);
}
int port = uri.getPort();
if (port == -1) {
port = SERVICE_BK_PORT;
}
return host + ":" + port;
}

private final String serviceName;
Expand Down
147 changes: 147 additions & 0 deletions bookkeeper-common/src/test/java/org/apache/bookkeeper/common/net/ServiceURITest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,14 @@ public void testMissingServiceName() {
null, new String[0], null, new String[] { "localhost:2181" }, "/path/to/namespace");
}

@Test
public void testMissingServiceNameIPV6() {
String serviceUri = "//[fec0:0:0:ffff::1]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
null, new String[0], null, new String[] { "[fec0:0:0:ffff::1]:2181" }, "/path/to/namespace");
}

@Test
public void testEmptyPath() {
String serviceUri = "bk://localhost:2181";
Expand All @@ -102,6 +110,14 @@ public void testEmptyPath() {
"bk", new String[0], null, new String[] { "localhost:2181" }, "");
}

@Test
public void testEmptyPathIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:2181";
assertServiceUri(
serviceUri,
"bk", new String[0], null, new String[] { "[fec0:0:0:ffff::1]:2181" }, "");
}

@Test
public void testRootPath() {
String serviceUri = "bk://localhost:2181/";
Expand All @@ -110,6 +126,14 @@ public void testRootPath() {
"bk", new String[0], null, new String[] { "localhost:2181" }, "/");
}

@Test
public void testRootPathIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:2181/";
assertServiceUri(
serviceUri,
"bk", new String[0], null, new String[] { "[fec0:0:0:ffff::1]:2181" }, "/");
}

@Test
public void testUserInfo() {
String serviceUri = "bk://bookkeeper@localhost:2181/path/to/namespace";
Expand All @@ -122,6 +146,18 @@ public void testUserInfo() {
"/path/to/namespace");
}

@Test
public void testUserInfoIPV6() {
String serviceUri = "bk://bookkeeper@[fec0:0:0:ffff::1]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
"bookkeeper",
new String[] { "[fec0:0:0:ffff::1]:2181" },
"/path/to/namespace");
}

@Test
public void testMultipleHostsSemiColon() {
String serviceUri = "bk://host1:2181;host2:2181;host3:2181/path/to/namespace";
Expand All @@ -134,6 +170,19 @@ public void testMultipleHostsSemiColon() {
"/path/to/namespace");
}

@Test
public void testMultipleHostsSemiColonIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:2181;[fec0:0:0:ffff::2]:2181;[fec0:0:0:ffff::3]:2181" +
"/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:2181", "[fec0:0:0:ffff::2]:2181", "[fec0:0:0:ffff::3]:2181" },
"/path/to/namespace");
}

@Test
public void testMultipleHostsComma() {
String serviceUri = "bk://host1:2181,host2:2181,host3:2181/path/to/namespace";
Expand All @@ -146,6 +195,19 @@ public void testMultipleHostsComma() {
"/path/to/namespace");
}

@Test
public void testMultipleHostsCommaIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:2181,[fec0:0:0:ffff::2]:2181,[fec0:0:0:ffff::3]:2181" +
"/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:2181", "[fec0:0:0:ffff::2]:2181", "[fec0:0:0:ffff::3]:2181" },
"/path/to/namespace");
}

@Test
public void testMultipleHostsWithoutPorts() {
String serviceUri = "bk://host1,host2,host3/path/to/namespace";
Expand All @@ -158,6 +220,18 @@ public void testMultipleHostsWithoutPorts() {
"/path/to/namespace");
}

@Test
public void testMultipleHostsWithoutPortsIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1],[fec0:0:0:ffff::2],[fec0:0:0:ffff::3]/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:4181", "[fec0:0:0:ffff::2]:4181", "[fec0:0:0:ffff::3]:4181" },
"/path/to/namespace");
}

@Test
public void testMultipleHostsMixedPorts() {
String serviceUri = "bk://host1:3181,host2,host3:2181/path/to/namespace";
Expand All @@ -170,6 +244,18 @@ public void testMultipleHostsMixedPorts() {
"/path/to/namespace");
}

@Test
public void testMultipleHostsMixedPortsIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:3181,[fec0:0:0:ffff::2],[fec0:0:0:ffff::3]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:3181", "[fec0:0:0:ffff::2]:4181", "[fec0:0:0:ffff::3]:2181" },
"/path/to/namespace");
}

@Test
public void testMultipleHostsMixed() {
String serviceUri = "bk://host1:2181,host2,host3:2181/path/to/namespace";
Expand All @@ -182,6 +268,18 @@ public void testMultipleHostsMixed() {
"/path/to/namespace");
}

@Test
public void testMultipleHostsMixedIPV6() {
String serviceUri = "bk://[fec0:0:0:ffff::1]:2181,[fec0:0:0:ffff::2],[fec0:0:0:ffff::3]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:2181", "[fec0:0:0:ffff::2]:4181", "[fec0:0:0:ffff::3]:2181" },
"/path/to/namespace");
}

@Test
public void testUserInfoWithMultipleHosts() {
String serviceUri = "bk://bookkeeper@host1:2181;host2:2181;host3:2181/path/to/namespace";
Expand All @@ -194,6 +292,19 @@ public void testUserInfoWithMultipleHosts() {
"/path/to/namespace");
}

@Test
public void testUserInfoWithMultipleHostsIPV6() {
String serviceUri = "bk://bookkeeper@[fec0:0:0:ffff::1]:2181;[fec0:0:0:ffff::2]:2181;" +
"[fec0:0:0:ffff::3]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[0],
"bookkeeper",
new String[] { "[fec0:0:0:ffff::1]:2181", "[fec0:0:0:ffff::2]:2181", "[fec0:0:0:ffff::3]:2181" },
"/path/to/namespace");
}

@Test
public void testServiceInfoPlus() {
String serviceUri = "bk+ssl://host:2181/path/to/namespace";
Expand All @@ -206,6 +317,18 @@ public void testServiceInfoPlus() {
"/path/to/namespace");
}

@Test
public void testServiceInfoPlusIPV6() {
String serviceUri = "bk+ssl://[fec0:0:0:ffff::1]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk",
new String[] { "ssl" },
null,
new String[] { "[fec0:0:0:ffff::1]:2181" },
"/path/to/namespace");
}

@Test
public void testServiceInfoMinus() {
String serviceUri = "bk-ssl://host:2181/path/to/namespace";
Expand All @@ -218,6 +341,18 @@ public void testServiceInfoMinus() {
"/path/to/namespace");
}

@Test
public void testServiceInfoMinusIPV6() {
String serviceUri = "bk-ssl://[fec0:0:0:ffff::1]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"bk-ssl",
new String[0],
null,
new String[] { "[fec0:0:0:ffff::1]:2181" },
"/path/to/namespace");
}

@Test
public void testServiceInfoDlogMinus() {
String serviceUri = "distributedlog-bk://host:2181/path/to/namespace";
Expand All @@ -230,4 +365,16 @@ public void testServiceInfoDlogMinus() {
"/path/to/namespace");
}

@Test
public void testServiceInfoDlogMinusIPV6() {
String serviceUri = "distributedlog-bk://[fec0:0:0:ffff::1]:2181/path/to/namespace";
assertServiceUri(
serviceUri,
"distributedlog",
new String[] { "bk" },
null,
new String[] { "[fec0:0:0:ffff::1]:2181" },
"/path/to/namespace");
}

}
3 changes: 2 additions & 1 deletion bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.net.URI;
import org.apache.bookkeeper.common.allocator.ByteBufAllocatorBuilder;
import org.apache.bookkeeper.common.allocator.ByteBufAllocatorWithOomHandler;
import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.MetadataBookieDriver;
Expand Down Expand Up @@ -54,7 +55,7 @@ public static MetadataBookieDriver createMetadataDriver(ServerConfiguration conf
}

MetadataBookieDriver driver = MetadataDrivers.getBookieDriver(
URI.create(metadataServiceUriStr));
ServiceURI.create(metadataServiceUriStr).getUri());
driver.initialize(conf, statsLogger.scope(BOOKIE_SCOPE));
return driver;
} catch (MetadataException me) {
Expand Down
3 changes: 2 additions & 1 deletion ...per-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import lombok.Cleanup;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.LedgerManager.LedgerRange;
Expand Down Expand Up @@ -301,7 +302,7 @@ private static MetadataBookieDriver instantiateMetadataDriver(ServerConfiguratio
throws BookieException {
try {
String metadataServiceUriStr = conf.getMetadataServiceUri();
MetadataBookieDriver driver = MetadataDrivers.getBookieDriver(URI.create(metadataServiceUriStr));
MetadataBookieDriver driver = MetadataDrivers.getBookieDriver(ServiceURI.create(metadataServiceUriStr).getUri());
driver.initialize(
conf,
statsLogger);
Expand Down
3 changes: 2 additions & 1 deletion bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import org.apache.bookkeeper.client.api.OpenBuilder;
import org.apache.bookkeeper.client.api.WriteFlag;
import org.apache.bookkeeper.common.allocator.ByteBufAllocatorBuilder;
import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.common.util.ReflectionUtils;
Expand Down Expand Up @@ -442,7 +443,7 @@ public BookKeeper(ClientConfiguration conf, ZooKeeper zk, EventLoopGroup eventLo
try {
String metadataServiceUriStr = conf.getMetadataServiceUri();
if (null != metadataServiceUriStr) {
this.metadataDriver = MetadataDrivers.getClientDriver(URI.create(metadataServiceUriStr));
this.metadataDriver = MetadataDrivers.getClientDriver(ServiceURI.create(metadataServiceUriStr).getUri());
} else {
checkNotNull(zkc, "No external zookeeper provided when no metadata service uri is found");
this.metadataDriver = MetadataDrivers.getClientDriver("zk");
Expand Down
3 changes: 2 additions & 1 deletion ...eeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManagerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.common.util.ReflectionUtils;
import org.apache.bookkeeper.conf.AbstractConfiguration;
import org.apache.bookkeeper.meta.LayoutManager.LedgerLayoutExistsException;
Expand Down Expand Up @@ -155,7 +156,7 @@ public static LedgerManagerFactory newLedgerManagerFactory(
}
ledgerRootPath = conf.getZkLedgersRootPath();
} else {
URI metadataServiceUri = URI.create(metadataServiceUriStr);
URI metadataServiceUri = ServiceURI.create(metadataServiceUriStr).getUri();
factoryClass = ZKMetadataDriverBase.resolveLedgerManagerFactory(metadataServiceUri);
ledgerRootPath = metadataServiceUri.getPath();
}
Expand Down
Loading