Skip to content

Commit afff124

Browse files
authored
IGNITE-27216 Added capturing of cluster node certificates during join process (#12546)
1 parent 6bca46b commit afff124

File tree

7 files changed

+253
-1
lines changed

7 files changed

+253
-1
lines changed

modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ public final class IgniteNodeAttributes {
150150
/** V2 security subject for authenticated node. */
151151
public static final String ATTR_SECURITY_SUBJECT_V2 = ATTR_PREFIX + ".security.subject.v2";
152152

153+
/** Node certificates the connection was established with. */
154+
public static final String ATTR_NODE_CERTIFICATES = ATTR_PREFIX + ".security.certificates";
155+
153156
/** Client mode flag. */
154157
public static final String ATTR_CLIENT_MODE = ATTR_PREFIX + ".cache.client";
155158

modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2301,6 +2301,9 @@ private void processNodeAddFinishedMessage(TcpDiscoveryNodeAddFinishedMessage ms
23012301
}
23022302

23032303
locNode.setAttributes(msg.clientNodeAttributes());
2304+
2305+
clearNodeSensitiveData(locNode);
2306+
23042307
locNode.visible(true);
23052308

23062309
long topVer = msg.topologyVersion();
@@ -2356,6 +2359,8 @@ else if (log.isDebugEnabled())
23562359
assert topVer > 0 : msg;
23572360

23582361
if (!node.visible()) {
2362+
clearNodeSensitiveData(node);
2363+
23592364
node.order(topVer);
23602365
node.visible(true);
23612366

modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@
178178
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
179179
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
180180
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
181+
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
181182
import static org.apache.ignite.internal.cluster.DistributedConfigurationUtils.CONN_DISABLED_BY_ADMIN_ERR_MSG;
182183
import static org.apache.ignite.internal.cluster.DistributedConfigurationUtils.newConnectionEnabledProperty;
183184
import static org.apache.ignite.internal.processors.security.SecurityUtils.authenticateLocalNode;
@@ -1134,7 +1135,7 @@ private void joinTopology() throws IgniteSpiException {
11341135
leavingNodes.clear();
11351136
failedNodesMsgSent.clear();
11361137

1137-
locNode.attributes().remove(IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS);
1138+
clearNodeSensitiveData(locNode);
11381139

11391140
locNode.order(1);
11401141
locNode.internalOrder(1);
@@ -2443,6 +2444,18 @@ private void processMessageFailedNodes(TcpDiscoveryAbstractMessage msg) {
24432444
}
24442445
}
24452446

2447+
/** */
2448+
private static void enrichNodeWithAttribute(TcpDiscoveryNode node, String attrName, @Nullable Object attrVal) {
2449+
if (attrVal == null)
2450+
return;
2451+
2452+
Map<String, Object> attrs = new HashMap<>(node.getAttributes());
2453+
2454+
attrs.put(attrName, attrVal);
2455+
2456+
node.setAttributes(attrs);
2457+
}
2458+
24462459
/** */
24472460
private static WorkersRegistry getWorkerRegistry(TcpDiscoverySpi spi) {
24482461
return spi.ignite() instanceof IgniteEx ? ((IgniteEx)spi.ignite()).context().workersRegistry() : null;
@@ -5298,6 +5311,8 @@ private void processNodeAddFinishedMessage(TcpDiscoveryNodeAddFinishedMessage ms
52985311
if (msg.verified()) {
52995312
assert topVer > 0 : "Invalid topology version: " + msg;
53005313

5314+
clearNodeSensitiveData(node);
5315+
53015316
if (node.order() == 0)
53025317
node.order(topVer);
53035318

@@ -7072,6 +7087,11 @@ else if (e.hasCause(ObjectStreamException.class) ||
70727087
else if (msg instanceof TcpDiscoveryJoinRequestMessage) {
70737088
TcpDiscoveryJoinRequestMessage req = (TcpDiscoveryJoinRequestMessage)msg;
70747089

7090+
// Current node holds connection with the node that is joining the cluster. Therefore, it can
7091+
// save certificates with which the connection was established to joining node attributes.
7092+
if (spi.nodeAuth != null && nodeId.equals(req.node().id()))
7093+
enrichNodeWithAttribute(req.node(), ATTR_NODE_CERTIFICATES, ses.extractCertificates());
7094+
70757095
if (!req.responded()) {
70767096
boolean ok = processJoinRequestMessage(req, clientMsgWrk);
70777097

modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.ArrayList;
2626
import java.util.Collection;
2727
import java.util.Collections;
28+
import java.util.HashMap;
2829
import java.util.List;
2930
import java.util.Map;
3031
import java.util.UUID;
@@ -53,6 +54,8 @@
5354

5455
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_METRICS_QNT_WARN;
5556
import static org.apache.ignite.IgniteSystemProperties.getInteger;
57+
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
58+
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
5659
import static org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.DFLT_DISCOVERY_METRICS_QNT_WARN;
5760

5861
/**
@@ -401,6 +404,16 @@ protected boolean checkAckTimeout(long ackTimeout) {
401404
return true;
402405
}
403406

407+
/** */
408+
protected void clearNodeSensitiveData(TcpDiscoveryNode node) {
409+
Map<String, Object> attrs = new HashMap<>(node.attributes());
410+
411+
attrs.remove(ATTR_NODE_CERTIFICATES);
412+
attrs.remove(ATTR_SECURITY_CREDENTIALS);
413+
414+
node.setAttributes(attrs);
415+
}
416+
404417
/** */
405418
public void processMsgCacheMetrics(TcpDiscoveryMetricsUpdateMessage msg, long tsNanos) {
406419
for (Map.Entry<UUID, TcpDiscoveryMetricsUpdateMessage.MetricsSet> e : msg.metrics().entrySet()) {

modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIoSession.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
import java.io.StreamCorruptedException;
2828
import java.net.Socket;
2929
import java.nio.ByteBuffer;
30+
import java.security.cert.Certificate;
31+
import javax.net.ssl.SSLPeerUnverifiedException;
32+
import javax.net.ssl.SSLSocket;
3033
import org.apache.ignite.IgniteCheckedException;
3134
import org.apache.ignite.IgniteException;
3235
import org.apache.ignite.internal.direct.DirectMessageReader;
@@ -36,6 +39,7 @@
3639
import org.apache.ignite.plugin.extensions.communication.Message;
3740
import org.apache.ignite.plugin.extensions.communication.MessageSerializer;
3841
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
42+
import org.jetbrains.annotations.Nullable;
3943

4044
import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.makeMessageType;
4145

@@ -206,6 +210,21 @@ <T> T readMessage() throws IgniteCheckedException, IOException {
206210
}
207211
}
208212

213+
/** @return SSL certificate this session is established with. {@code null} if SSL is disabled or certificate validation failed. */
214+
@Nullable Certificate[] extractCertificates() {
215+
if (!spi.isSslEnabled())
216+
return null;
217+
218+
try {
219+
return ((SSLSocket)sock).getSession().getPeerCertificates();
220+
}
221+
catch (SSLPeerUnverifiedException e) {
222+
U.error(spi.log, "Failed to extract discovery IO session certificates", e);
223+
224+
return null;
225+
}
226+
}
227+
209228
/**
210229
* Serializes a discovery message into a byte array.
211230
*
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.ignite.internal.processors.security;
19+
20+
import java.security.Permissions;
21+
import java.security.cert.Certificate;
22+
import java.security.cert.X509Certificate;
23+
import java.util.Arrays;
24+
import java.util.Collection;
25+
import java.util.UUID;
26+
import java.util.concurrent.ConcurrentLinkedQueue;
27+
import java.util.concurrent.CountDownLatch;
28+
import org.apache.ignite.Ignite;
29+
import org.apache.ignite.IgniteCheckedException;
30+
import org.apache.ignite.cluster.ClusterNode;
31+
import org.apache.ignite.configuration.IgniteConfiguration;
32+
import org.apache.ignite.internal.GridKernalContext;
33+
import org.apache.ignite.internal.IgniteEx;
34+
import org.apache.ignite.internal.processors.security.impl.TestSecurityData;
35+
import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider;
36+
import org.apache.ignite.internal.processors.security.impl.TestSecurityProcessor;
37+
import org.apache.ignite.internal.util.typedef.G;
38+
import org.apache.ignite.plugin.security.SecurityCredentials;
39+
import org.apache.ignite.plugin.security.SecurityPermissionSet;
40+
import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
41+
import org.apache.ignite.testframework.GridTestUtils;
42+
import org.junit.Test;
43+
44+
import static java.util.concurrent.TimeUnit.MILLISECONDS;
45+
import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
46+
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CERTIFICATES;
47+
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
48+
import static org.apache.ignite.plugin.security.SecurityPermission.JOIN_AS_SERVER;
49+
import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.NO_PERMISSIONS;
50+
import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.systemPermissions;
51+
52+
/** */
53+
public class NodeConnectionCertificateCapturingTest extends AbstractSecurityTest {
54+
/** */
55+
private static final Collection<AuthenticationEvent> NODE_AUTHENTICATION_EVENTS = new ConcurrentLinkedQueue<>();
56+
57+
/** */
58+
@Test
59+
public void testNodeConnectionCertificateCapturing() throws Exception {
60+
checkNewNodeAuthenticationByClusterNodes(0, false, 1);
61+
checkNewNodeAuthenticationByClusterNodes(1, false, 2);
62+
checkNewNodeAuthenticationByClusterNodes(2, false, 3);
63+
checkNewNodeAuthenticationByClusterNodes(3, true, 3);
64+
65+
// Checks nodes restart.
66+
stopGrid(2);
67+
stopGrid(3);
68+
69+
checkNewNodeAuthenticationByClusterNodes(3, true, 2);
70+
checkNewNodeAuthenticationByClusterNodes(2, false, 3);
71+
72+
// Checks client node reconnect.
73+
NODE_AUTHENTICATION_EVENTS.clear();
74+
75+
CountDownLatch cliNodeReconnectedLatch = new CountDownLatch(1);
76+
77+
grid(3).events().localListen(evt -> {
78+
cliNodeReconnectedLatch.countDown();
79+
80+
return true;
81+
}, EVT_CLIENT_NODE_RECONNECTED);
82+
83+
grid(0).context().discovery().failNode(grid(3).localNode().id(), "test");
84+
85+
assertTrue(cliNodeReconnectedLatch.await(getTestTimeout(), MILLISECONDS));
86+
87+
checkNodeAuthenticationByClusterNodes(3, grid(3).localNode().id(), true, 3);
88+
}
89+
90+
/** */
91+
private void checkNewNodeAuthenticationByClusterNodes(int authNodeIdx, boolean isClient, int expAuthCnt) throws Exception {
92+
NODE_AUTHENTICATION_EVENTS.clear();
93+
94+
UUID authNodeId = startGrid(authNodeIdx, isClient).cluster().localNode().id();
95+
96+
checkNodeAuthenticationByClusterNodes(authNodeIdx, authNodeId, isClient, expAuthCnt);
97+
}
98+
99+
/** */
100+
private void checkNodeAuthenticationByClusterNodes(int authNodeIdx, UUID authNodeId, boolean isClient, int expAuthCnt) {
101+
assertEquals(expAuthCnt, NODE_AUTHENTICATION_EVENTS.size());
102+
103+
for (AuthenticationEvent auth : NODE_AUTHENTICATION_EVENTS) {
104+
if (auth.clusterNodeId.equals(authNodeId))
105+
assertNull(auth.certs);
106+
else {
107+
assertEquals(2, auth.certs.length);
108+
109+
X509Certificate cert = (X509Certificate)auth.certs[0];
110+
111+
assertEquals(isClient ? "CN=client" : "CN=node0" + (authNodeIdx + 1), cert.getSubjectDN().getName());
112+
}
113+
}
114+
115+
for (Ignite ignite : G.allGrids()) {
116+
for (ClusterNode node : ignite.cluster().nodes()) {
117+
assertNull(((TcpDiscoveryNode)node).getAttributes().get(ATTR_SECURITY_CREDENTIALS));
118+
assertNull(((TcpDiscoveryNode)node).getAttributes().get(ATTR_NODE_CERTIFICATES));
119+
}
120+
}
121+
}
122+
123+
/** */
124+
private IgniteEx startGrid(int idx, boolean isClient) throws Exception {
125+
String login = getTestIgniteInstanceName(idx);
126+
127+
IgniteConfiguration cfg = getConfiguration(
128+
login,
129+
new SecurityPluginProvider(
130+
login,
131+
"",
132+
isClient ? NO_PERMISSIONS : systemPermissions(JOIN_AS_SERVER),
133+
null,
134+
true
135+
)).setClientMode(isClient);
136+
137+
cfg.setSslContextFactory(GridTestUtils.sslTrustedFactory(isClient ? "client" : "node0" + (idx + 1), "trustboth"));
138+
139+
return startGrid(cfg);
140+
}
141+
142+
/** */
143+
private static class SecurityPluginProvider extends TestSecurityPluginProvider {
144+
/** */
145+
SecurityPluginProvider(
146+
String login,
147+
String pwd,
148+
SecurityPermissionSet perms,
149+
Permissions sandboxPerms,
150+
boolean globalAuth,
151+
TestSecurityData... clientData
152+
) {
153+
super(login, pwd, perms, sandboxPerms, globalAuth, clientData);
154+
}
155+
156+
/** {@inheritDoc} */
157+
@Override protected GridSecurityProcessor securityProcessor(GridKernalContext ctx) {
158+
return new TestSecurityProcessor(
159+
ctx,
160+
new TestSecurityData(login, pwd, perms, sandboxPerms),
161+
Arrays.asList(clientData),
162+
globalAuth
163+
) {
164+
@Override public SecurityContext authenticateNode(
165+
ClusterNode node,
166+
SecurityCredentials cred
167+
) throws IgniteCheckedException {
168+
NODE_AUTHENTICATION_EVENTS.add(new AuthenticationEvent(ctx.localNodeId(), node.attribute(ATTR_NODE_CERTIFICATES)));
169+
170+
return super.authenticateNode(node, cred);
171+
}
172+
};
173+
}
174+
}
175+
176+
/** */
177+
private static class AuthenticationEvent {
178+
/** */
179+
UUID clusterNodeId;
180+
181+
/** */
182+
Certificate[] certs;
183+
184+
/** */
185+
AuthenticationEvent(UUID clusterNodeId, Certificate[] certs) {
186+
this.clusterNodeId = clusterNodeId;
187+
this.certs = certs;
188+
}
189+
}
190+
}

modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.ignite.internal.processors.security.IgniteSecurityProcessorTest;
2121
import org.apache.ignite.internal.processors.security.InvalidServerTest;
22+
import org.apache.ignite.internal.processors.security.NodeConnectionCertificateCapturingTest;
2223
import org.apache.ignite.internal.processors.security.NodeSecurityContextPropagationTest;
2324
import org.apache.ignite.internal.processors.security.SecurityContextInternalFuturePropagationTest;
2425
import org.apache.ignite.internal.processors.security.cache.CacheOperationPermissionCheckTest;
@@ -143,6 +144,7 @@
143144
NodeJoinPermissionsTest.class,
144145
ActivationOnJoinWithoutPermissionsWithPersistenceTest.class,
145146
SecurityContextInternalFuturePropagationTest.class,
147+
NodeConnectionCertificateCapturingTest.class,
146148
})
147149
public class SecurityTestSuite {
148150
/** */

0 commit comments

Comments
 (0)