chore: Minor improvements of Infinispan (v3)

apiml-package/src/main/resources/bin/start.sh

@@ -334,6 +334,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${APIML_CODE} ${JAVA_BIN_DIR}java \
     -Djgroups.bind.address=${ZWE_components_caching_service_storage_infinispan_jgroups_host:-${ZWE_configs_storage_infinispan_jgroups_host:-${ZWE_haInstance_hostname:-localhost}}} \
     -Djgroups.bind.port=${ZWE_components_caching_service_storage_infinispan_jgroups_port:-${ZWE_configs_storage_infinispan_jgroups_port:-7600}} \
     -Djgroups.keyExchange.port=${ZWE_components_caching_service_storage_infinispan_jgroups_keyExchange_port:-${ZWE_configs_storage_infinispan_jgroups_keyExchange_port:-7601}} \
+    -Djgroups.keyExchange.socketTimeout=${ZWE_components_caching_service_storage_infinispan_jgroups_keyExchange_socketTimeout:-${ZWE_configs_storage_infinispan_jgroups_keyExchange_socketTimeout:-5000}} \
     -Djgroups.tcp.diag.enabled=${ZWE_components_caching_service_storage_infinispan_jgroups_tcp_diag_enabled:-${ZWE_configs_storage_infinispan_jgroups_tcp_diag_enabled:-false}} \
     -Dloader.path=${APIML_LOADER_PATH} \
     -Dlogging.charset.console=${ZOWE_CONSOLE_LOG_CHARSET} \

caching-service-package/src/main/resources/bin/start.sh

@@ -137,6 +137,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} ${JAVA_BIN_DIR}java \
   -Djgroups.bind.port=${ZWE_configs_storage_infinispan_jgroups_port:-7600} \
   -Djgroups.keyExchange.port=${ZWE_configs_storage_infinispan_jgroups_keyExchange_port:-7601} \
   -Djgroups.tcp.diag.enabled=${ZWE_configs_storage_infinispan_jgroups_tcp_diag_enabled:-false} \
+  -Djgroups.keyExchange.socketTimeout=${ZWE_configs_storage_infinispan_jgroups_keyExchange_socketTimeout:-5000} \
   -Dcaching.storage.infinispan.initialHosts=${ZWE_configs_storage_infinispan_initialHosts:-localhost[7600]} \
   -Dserver.address=${ZWE_configs_zowe_network_server_listenAddresses_0:-${ZWE_zowe_network_server_listenAddresses_0:-"0.0.0.0"}} \
   -Dserver.ssl.enabled=${ZWE_configs_server_ssl_enabled:-true}  \

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/ApimlSslKeyExchange.java

@@ -0,0 +1,290 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.caching.service.infinispan;
+
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Delegate;
+import lombok.extern.slf4j.Slf4j;
+import org.jgroups.Address;
+import org.jgroups.protocols.SSL_KEY_EXCHANGE;
+import org.jgroups.stack.IpAddress;
+
+import javax.net.ssl.*;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.security.KeyManagementException;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class ApimlSslKeyExchange extends SSL_KEY_EXCHANGE {
+
+    private static final ThreadLocal<List<Throwable>> EXCEPTIONS = new ThreadLocal<>();
+
+    private static void addException(Exception e) {
+        List<Throwable> exceptionList = EXCEPTIONS.get();
+        if (exceptionList == null) {
+            exceptionList = new ArrayList<>();
+        }
+        exceptionList.add(e);
+        EXCEPTIONS.set(exceptionList);
+    }
+
+    String toString(Throwable t) {
+        List<Throwable> stack = new ArrayList<>();
+        Throwable previous;
+        do {
+            stack.add(t);
+            previous = t;
+            t = t.getCause();
+        } while ((t != null) && (t != previous));
+        return stack.stream().map(Throwable::toString).collect(Collectors.joining(": "));
+    }
+
+    void printError(String message, List<Throwable> exceptionList) {
+        log.error("{}: {}", message, exceptionList.stream().map(this::toString).collect(Collectors.joining(", ")));
+    }
+
+    void printError(String message) {
+        List<Throwable> exceptionList = EXCEPTIONS.get();
+        if (exceptionList != null) {
+            printError(message, exceptionList);
+            EXCEPTIONS.remove();
+        }
+    }
+
+    void decorate(Exception e) {
+        List<Throwable> exceptionList = EXCEPTIONS.get();
+        if (exceptionList != null) {
+            Iterator<Throwable> iterator = exceptionList.iterator();
+            if ((e.getCause() == null) || (e.getCause() == e)) {
+                e.initCause(iterator.next());
+            }
+            while (iterator.hasNext()) {
+                e.addSuppressed(iterator.next());
+            }
+        }
+    }
+
+    protected SSLServerSocket createServerSocket() throws Exception {
+        try {
+            return super.createServerSocket();
+        } catch (Exception e) {
+            decorate(e);
+            throw e;
+        } finally {
+            printError("Cannot create server socket");
+        }
+    }
+
+    @Override
+    protected SSLSocket createSocketTo(Address target) throws Exception {
+        try {
+            return super.createSocketTo(target);
+        } catch (Exception e) {
+            decorate(e);
+            throw e;
+        } finally {
+            printError("Cannot create socket to remote address");
+        }
+    }
+
+    @Override
+    protected SSLSocket createSocketTo(IpAddress dest, SSLSocketFactory sslSocketFactory) {
+        try {
+            return super.createSocketTo(dest, sslSocketFactory);
+        } catch (RuntimeException re) {
+            decorate(re);
+            throw re;
+        } finally {
+            printError("Cannot create socket to remote address");
+        }
+    }
+
+    private SSLContext update(SSLContext context) {
+        return new SSLContextWrapper(
+            new SSLContextSpiWrapper(
+                null,
+                new SSLSocketFactoryWrapper(context.getSocketFactory()),
+                new SSLServerSocketFactoryWrapper(context.getServerSocketFactory())
+            ),
+            context
+        );
+    }
+
+    @Override
+    public void init() throws Exception {
+        synchronized (ApimlSslKeyExchange.class) {
+            boolean update = (client_ssl_ctx == null || server_ssl_ctx == null);
+            super.init();
+            if (update) {
+                super.client_ssl_ctx = update(super.client_ssl_ctx);
+                super.server_ssl_ctx = update(super.server_ssl_ctx);
+            }
+        }
+    }
+
+    @Override
+    public SSL_KEY_EXCHANGE setClientSSLContext(SSLContext client_ssl_ctx) {
+        return super.setClientSSLContext(update(client_ssl_ctx));
+    }
+
+    @Override
+    public SSL_KEY_EXCHANGE setServerSSLContext(SSLContext server_ssl_ctx) {
+        return super.setServerSSLContext(update(server_ssl_ctx));
+    }
+
+    @RequiredArgsConstructor
+    static class SSLSocketFactoryWrapper extends SSLSocketFactory {
+
+        @Delegate
+        private final SSLSocketFactory original;
+
+        @Override
+        public String[] getDefaultCipherSuites() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public String[] getSupportedCipherSuites() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException, UnknownHostException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public Socket createSocket(InetAddress host, int port) throws IOException {
+            try {
+                return original.createSocket(host, port);
+            } catch (IOException | RuntimeException e) {
+                addException(e);
+                throw e;
+            }
+        }
+
+        @Override
+        public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+    }
+
+    @RequiredArgsConstructor
+    static class SSLServerSocketFactoryWrapper extends SSLServerSocketFactory {
+
+        @Delegate
+        private final SSLServerSocketFactory original;
+
+        @Override
+        public String[] getDefaultCipherSuites() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public String[] getSupportedCipherSuites() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public ServerSocket createServerSocket(int port) throws IOException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public ServerSocket createServerSocket(int port, int backlog) throws IOException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        public ServerSocket createServerSocket(int port, int backlog, InetAddress ifAddress) throws IOException {
+            try {
+                return original.createServerSocket(port, backlog, ifAddress);
+            } catch (IOException | RuntimeException e) {
+                addException(e);
+                throw e;
+            }
+        }
+
+    }
+
+    @RequiredArgsConstructor
+    static class SSLContextSpiWrapper extends SSLContextSpi {
+
+        @Delegate
+        private final SSLContextSpi original;
+        private final SSLSocketFactory sslSocketFactory;
+        private final SSLServerSocketFactory sslServerSocketFactory;
+
+        @Override
+        protected void engineInit(KeyManager[] km, TrustManager[] tm, SecureRandom sr) throws KeyManagementException {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        protected SSLSocketFactory engineGetSocketFactory() {
+            return this.sslSocketFactory;
+        }
+
+        @Override
+        protected SSLServerSocketFactory engineGetServerSocketFactory() {
+            return this.sslServerSocketFactory;
+        }
+
+        @Override
+        protected SSLEngine engineCreateSSLEngine() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        protected SSLEngine engineCreateSSLEngine(String host, int port) {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        protected SSLSessionContext engineGetServerSessionContext() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+        @Override
+        protected SSLSessionContext engineGetClientSessionContext() {
+            throw new IllegalStateException("Not implemented");
+        }
+
+    }
+
+    static class SSLContextWrapper extends SSLContext {
+
+        SSLContextWrapper(SSLContextSpi contextSpi, SSLContext original) {
+            super(contextSpi, original.getProvider(), original.getProtocol());
+        }
+
+    }
+
+}

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java

@@ -89,6 +89,9 @@ public class InfinispanConfig implements InitializingBean {
     @Value("${jgroups.bind.address}")
     private String address;
 
+    @Value("${jgroups.keyExchange.socketTimeout:5000}")
+    private String keyExchangeSocketTimeout;
+
     @Value("${jgroups.keyExchange.port:7601}")
     private String keyExchangePort;
 
@@ -98,9 +101,6 @@ public class InfinispanConfig implements InitializingBean {
     @Value("${attlsEnabledOnInfinispanTest:${server.attlsServer.enabled:false}}")
     private boolean isServerAttlsEnabled;
 
-    @Value("${apiml.service.hostname:localhost}")
-    private String hostname;
-
     @Value("${caching.storage.infinispan.distributedSyncTimeoutSecs:360}")
     private int distributedSyncTimeout;
 
@@ -181,6 +181,7 @@ synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader, Applic
         System.setProperty("jgroups.tcpping.initial_hosts", initialHosts);
         System.setProperty("jgroups.bind.port", port);
         System.setProperty("jgroups.bind.address", address);
+        System.setProperty("jgroups.keyExchange.socketTimeout", keyExchangeSocketTimeout);
         System.setProperty("jgroups.keyExchange.port", keyExchangePort);
         System.setProperty("jgroups.tcp.diag.enabled", String.valueOf(Boolean.parseBoolean(tcpDiagEnabled)));
 

caching-service/src/main/resources/infinispan.xml

@@ -33,20 +33,23 @@
                     timeout_check_interval="1000"
             />
             <VERIFY_SUSPECT timeout="1000"/>
-            <SSL_KEY_EXCHANGE keystore_name="${infinispan.ssl.keyStore}"
+            <!-- replacement of SSL_KEY_EXCHANGE -->
+            <org.zowe.apiml.caching.service.infinispan.ApimlSslKeyExchange keystore_name="${infinispan.ssl.keyStore}"
                               keystore_type="${infinispan.ssl.keyStoreType}"
                               keystore_password="${infinispan.ssl.keyStorePassword}"
                               truststore_name="${infinispan.ssl.trustStore}"
                               truststore_type="${infinispan.ssl.trustStoreType}"
                               truststore_password="${infinispan.ssl.trustStorePassword}"
                               secret_key_algorithm="AES"
                               port="${jgroups.keyExchange.port}"
+                              port_range="0"
+                              socket_timeout="${jgroups.keyExchange.socketTimeout:5000}"
                               stack.combine="INSERT_AFTER"
                               stack.position="VERIFY_SUSPECT"
                               require_client_authentication="false"
             />
             <!-- Configures ASYM_ENCRYPT -->
-            <!-- The use_external_key_exchange = "true" attribute configures nodes to use the `SSL_KEY_EXCHANGE` protocol for certificate authentication. -->
+            <!-- The use_external_key_exchange = "true" attribute configures nodes to use the `org.zowe.apiml.caching.service.infinispan.ApimlSslKeyExchange` protocol for certificate authentication. -->
             <ASYM_ENCRYPT asym_keylength="2048"
                           asym_algorithm="RSA"
                           change_key_on_coord_leave="true"
@@ -56,7 +59,7 @@
                           sym_algorithm="AES/CBC/PKCS5Padding"
                           sym_iv_length="16"
                           stack.combine="INSERT_AFTER"
-                          stack.position="SSL_KEY_EXCHANGE"
+                          stack.position="org.zowe.apiml.caching.service.infinispan.ApimlSslKeyExchange"
             />
 
             <pbcast.NAKACK2/>