Merge pull request #1525 from marklogic/feature/cloud-reverse-proxy

Enhancements for connecting to ML Cloud

Author: rjrudin

marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java

@@ -143,9 +143,17 @@ public DatabaseClientBuilder withDigestAuth(String username, String password) {
 	}
 
 	public DatabaseClientBuilder withMarkLogicCloudAuth(String apiKey, String basePath) {
-		return withSecurityContextType(SECURITY_CONTEXT_TYPE_MARKLOGIC_CLOUD)
+		withSecurityContextType(SECURITY_CONTEXT_TYPE_MARKLOGIC_CLOUD)
 			.withCloudApiKey(apiKey)
 			.withBasePath(basePath);
+
+		// Assume sensible defaults for establishing an SSL connection. In the scenario where the user's JVM's
+		// truststore has a certificate matching that of the MarkLogic Cloud instance, this saves the user from having
+		// to configure anything except the API key and base path.
+		if (null == props.get(PREFIX + "sslProtocol") && null == props.get(PREFIX + "sslContext")) {
+			withSSLProtocol("default");
+		}
+		return this;
 	}
 
 	public DatabaseClientBuilder withKerberosAuth(String principal) {

marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java

@@ -18,6 +18,8 @@
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.DatabaseClientBuilder;
 import com.marklogic.client.DatabaseClientFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.X509TrustManager;
@@ -37,7 +39,8 @@
  */
 public class DatabaseClientPropertySource {
 
-	private final static String PREFIX = DatabaseClientBuilder.PREFIX;
+	private static final Logger logger = LoggerFactory.getLogger(DatabaseClientPropertySource.class);
+	private static final String PREFIX = DatabaseClientBuilder.PREFIX;
 
 	private final Function<String, Object> propertySource;
 
@@ -226,6 +229,18 @@ private X509TrustManager determineTrustManager() {
 			throw new IllegalArgumentException(
 				String.format("Trust manager must be an instance of %s", X509TrustManager.class.getName()));
 		}
+		// If the user chooses the "default" SSLContext, then it's already been initialized - but OkHttp still
+		// needs a separate X509TrustManager, so use the JVM's default trust manager. The assumption is that the
+		// default SSLContext was initialized with the JVM's default trust manager. A user can of course always override
+		// this by simply providing their own trust manager.
+		if ("default".equalsIgnoreCase((String) propertySource.apply(PREFIX + "sslProtocol"))) {
+			X509TrustManager defaultTrustManager = SSLUtil.getDefaultTrustManager();
+			if (logger.isDebugEnabled() && defaultTrustManager != null && defaultTrustManager.getAcceptedIssuers() != null) {
+				logger.debug("Count of accepted issuers in default trust manager: {}",
+					defaultTrustManager.getAcceptedIssuers().length);
+			}
+			return defaultTrustManager;
+		}
 		return null;
 	}
 

marklogic-client-api/src/main/java/com/marklogic/client/impl/SSLUtil.java

@@ -0,0 +1,45 @@
+package com.marklogic.client.impl;
+
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+
+public interface SSLUtil {
+
+	static X509TrustManager getDefaultTrustManager() {
+		return (X509TrustManager) getDefaultTrustManagers()[0];
+	}
+
+	/**
+	 * @return a non-empty array of TrustManager instances based on the JVM's default trust manager algorithm, with the
+	 * first trust manager guaranteed to be an instance of X509TrustManager.
+	 */
+	static TrustManager[] getDefaultTrustManagers() {
+		final String defaultAlgorithm = TrustManagerFactory.getDefaultAlgorithm();
+		TrustManagerFactory trustManagerFactory;
+		try {
+			trustManagerFactory = TrustManagerFactory.getInstance(defaultAlgorithm);
+		} catch (NoSuchAlgorithmException e) {
+			throw new RuntimeException("Unable to obtain trust manager factory using JVM's default trust manager algorithm: " + defaultAlgorithm, e);
+		}
+
+		try {
+			trustManagerFactory.init((KeyStore) null);
+		} catch (KeyStoreException e) {
+			throw new RuntimeException("Unable to initialize trust manager factory obtained using JVM's default trust manager algorithm: " + defaultAlgorithm
+				+ "; cause: " + e.getMessage(), e);
+		}
+
+		TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
+		if (trustManagers == null || trustManagers.length == 0) {
+			throw new RuntimeException("No trust managers found using the JVM's default trust manager algorithm: " + defaultAlgorithm);
+		}
+		if (!(trustManagers[0] instanceof X509TrustManager)) {
+			throw new RuntimeException("Default trust manager is not an X509TrustManager: " + trustManagers[0]);
+		}
+		return trustManagers;
+	}
+}

marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/MarkLogicCloudAuthenticationConfigurer.java

@@ -61,6 +61,7 @@ private Response callTokenEndpoint(MarkLogicCloudAuthContext securityContext) {
 		// Current assumption is that the SSL config provided for connecting to MarkLogic should also be applicable
 		// for connecting to MarkLogic Cloud's "/token" endpoint.
 		OkHttpUtil.configureSocketFactory(clientBuilder, securityContext.getSSLContext(), securityContext.getTrustManager());
+		OkHttpUtil.configureHostnameVerifier(clientBuilder, securityContext.getSSLHostnameVerifier());
 
 		if (logger.isInfoEnabled()) {
 			logger.info("Calling token endpoint at: " + tokenUrl);

marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java

@@ -3,6 +3,7 @@
 import com.marklogic.client.DatabaseClientFactory;
 import com.marklogic.client.impl.HTTPKerberosAuthInterceptor;
 import com.marklogic.client.impl.HTTPSamlAuthInterceptor;
+import com.marklogic.client.impl.SSLUtil;
 import okhttp3.ConnectionPool;
 import okhttp3.CookieJar;
 import okhttp3.Dns;
@@ -13,15 +14,11 @@
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -123,7 +120,7 @@ private static void configureSAMLAuth(DatabaseClientFactory.SAMLAuthContext saml
 	 * @param clientBuilder
 	 * @param sslVerifier
 	 */
-	private static void configureHostnameVerifier(OkHttpClient.Builder clientBuilder, DatabaseClientFactory.SSLHostnameVerifier sslVerifier) {
+	static void configureHostnameVerifier(OkHttpClient.Builder clientBuilder, DatabaseClientFactory.SSLHostnameVerifier sslVerifier) {
 		HostnameVerifier hostnameVerifier = null;
 		if (DatabaseClientFactory.SSLHostnameVerifier.ANY.equals(sslVerifier)) {
 			hostnameVerifier = (hostname, session) -> true;
@@ -169,25 +166,16 @@ static void configureSocketFactory(OkHttpClient.Builder clientBuilder, SSLContex
 	 * @param sslContext
 	 */
 	private static void initializeSslContext(OkHttpClient.Builder clientBuilder, SSLContext sslContext) {
+		TrustManager[] trustManagers = SSLUtil.getDefaultTrustManagers();
 		try {
-			TrustManagerFactory trustMgrFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-			trustMgrFactory.init((KeyStore) null);
-			TrustManager[] trustMgrs = trustMgrFactory.getTrustManagers();
-			if (trustMgrs == null || trustMgrs.length == 0) {
-				throw new IllegalArgumentException("no trust manager and could not get default trust manager");
-			}
-			if (!(trustMgrs[0] instanceof X509TrustManager)) {
-				throw new IllegalArgumentException("no trust manager and default is not an X509TrustManager");
-			}
-			sslContext.init(null, trustMgrs, null);
-			clientBuilder.sslSocketFactory(new SSLSocketFactoryDelegator(sslContext.getSocketFactory()), (X509TrustManager) trustMgrs[0]);
-		} catch (KeyStoreException e) {
-			throw new IllegalArgumentException("no trust manager and cannot initialize factory for default", e);
-		} catch (NoSuchAlgorithmException e) {
-			throw new IllegalArgumentException("no trust manager and no algorithm for default manager", e);
+			// In a future release, we may want to check if getSocketFactory() works already, implying that the
+			// SSLContext has already been initialized. However, if that's the case, then it's not guaranteed that
+			// the default trust manager is the appropriate one to pass to OkHttp.
+			sslContext.init(null, trustManagers, null);
 		} catch (KeyManagementException e) {
-			throw new IllegalArgumentException("no trust manager and cannot initialize context with default", e);
+			throw new RuntimeException("Unable to initialize SSLContext; cause: " + e.getMessage(), e);
 		}
+		clientBuilder.sslSocketFactory(new SSLSocketFactoryDelegator(sslContext.getSocketFactory()), (X509TrustManager) trustManagers[0]);
 	}
 
 	static class DnsImpl implements Dns {

marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientBuilderTest.java

@@ -107,6 +107,15 @@ void cloudWithBasePath() {
 			(DatabaseClientFactory.MarkLogicCloudAuthContext) bean.getSecurityContext();
 		assertEquals("my-key", context.getKey());
 		assertEquals("/my/path", bean.getBasePath());
+
+		assertNotNull(context.getSSLContext(), "If no sslProtocol or sslContext is set, the JVM's default SSL " +
+			"context should be used");
+
+		assertNotNull(context.getSSLContext().getSocketFactory(), "Since the JVM's default SSL context is expected " +
+			"to be used, it should already be initialized, and thus able to return a socket factory");
+
+		assertNotNull(context.getTrustManager(), "Since the JVM's default SSL context is used, the JVM's default " +
+			"trust manager should be used as well if the user doesn't provide their own");
 	}
 
 	@Test
@@ -174,6 +183,27 @@ void sslProtocol() {
 				"initialize the SSLContext before using it by using the JVM's default trust manager.");
 	}
 
+	@Test
+	void defaultSslProtocolAndNoTrustManager() {
+		bean = Common.newClientBuilder()
+			.withSSLProtocol("default")
+			.buildBean();
+
+		DatabaseClientFactory.SecurityContext context = bean.getSecurityContext();
+		assertNotNull(context);
+
+		SSLContext sslContext = context.getSSLContext();
+		assertNotNull(sslContext);
+		assertNotNull(sslContext.getSocketFactory(), "A protocol of 'default' should result in the JVM's default " +
+			"SSLContext being used, which is expected to have been initialized already and can thus return a socket " +
+			"factory");
+
+		assertNotNull(context.getTrustManager(), "If the user specifies a protocol of 'default' but does not " +
+			"provide a trust manager, the assumption is that the JVM's default trust manager should be used, thus " +
+			"saving the user from having to do the work of providing this themselves.");
+	}
+
+
 	@Test
 	void invalidSslProtocol() {
 		RuntimeException ex = assertThrows(RuntimeException.class, () -> Common.newClientBuilder()

marklogic-client-api/src/test/java/com/marklogic/client/test/MarkLogicCloudAuthenticationDebugger.java

@@ -5,13 +5,13 @@
 import com.marklogic.client.DatabaseClientFactory;
 import com.marklogic.client.io.JacksonHandle;
 
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
-
 /**
  * We don't yet have a way to run tests against a MarkLogic Cloud instance. In the meantime, this program and its
  * related Gradle task can be used for easy manual testing.
+ *
+ * For local testing against the ReverseProxyServer in the test-app project, which emulates MarkLogic Cloud, use
+ * "localhost" as the cloud host, "username:password" (often "admin:the admin password") as the apiKey, and
+ * "local/manage" as the basePath.
  */
 public class MarkLogicCloudAuthenticationDebugger {
 
@@ -20,11 +20,12 @@ public static void main(String[] args) throws Exception {
 		String apiKey = args[1];
 		String basePath = args[2];
 
+		// Expected to default to the JVM's default SSL context and default trust manager
 		DatabaseClient client = new DatabaseClientBuilder()
 			.withHost(cloudHost)
 			.withMarkLogicCloudAuth(apiKey, basePath)
-			.withSSLContext(SSLContext.getDefault())
-			.withTrustManager(Common.TRUST_ALL_MANAGER)
+			// Have to use "ANY", as the default is "COMMON", which won't work for our selfsigned cert
+			.withSSLHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY)
 			.build();
 
 		DatabaseClient.ConnectionResult result = client.checkConnection();

test-app/build.gradle

@@ -8,6 +8,10 @@ dependencies {
 	implementation "io.undertow:undertow-core:2.2.21.Final"
 	implementation "io.undertow:undertow-servlet:2.2.21.Final"
 	implementation "com.marklogic:ml-javaclient-util:4.4.0"
+	implementation 'org.slf4j:slf4j-api:1.7.36'
+	implementation 'ch.qos.logback:logback-classic:1.3.5'
+	implementation "com.fasterxml.jackson.core:jackson-databind:2.14.1"
+	implementation 'com.squareup.okhttp3:okhttp:4.10.0'
 }
 
 // See https://github.com/psxpaul/gradle-execfork-plugin for docs.
@@ -22,3 +26,13 @@ task runReverseProxyServer(type: com.github.psxpaul.task.JavaExecFork, dependsOn
 	standardOutput = file("$buildDir/reverse-proxy.log")
 	errorOutput = file("$buildDir/reverse-proxy-error.log")
 }
+
+task runBlockingReverseProxyServer(type: JavaExec) {
+	description = "Run the reverse proxy server so that it blocks and waits for requests; use ctrl-C to stop it. " +
+		"This is intended for manual testing with the reverse proxy server. If you wish to enable an HTTPS port that is 443" +
+		"or any value less than 1024, you will need to use sudo to run this - e.g. " +
+		"sudo ./gradlew runBlockingReverseProxyServer -PrpsHttpsPort=443 ."
+	classpath = sourceSets.main.runtimeClasspath
+	main = "com.marklogic.client.test.ReverseProxyServer"
+	args = [rpsMarkLogicServer, rpsProxyServer, rpsHttpPort, rpsHttpsPort]
+}

test-app/gradle.properties

@@ -6,3 +6,8 @@ mlRestPort=8012
 mlContentDatabaseName=java-unittest
 mlTdeValidationEnabled=false
 mlForestsPerHost=java-functest,2,java-unittest,3
+
+rpsMarkLogicServer=localhost
+rpsProxyServer=localhost
+rpsHttpPort=8020
+rpsHttpsPort=0