Skip to content

Commit e79b395

Browse files
anton-vinogradovanton-vinogradov
authored
IGNITE-26334 Implement a TopologyValidator based on DataCenterId (#12442)
1 parent 5901b7f commit e79b395

File tree

11 files changed

+540
-5
lines changed

11 files changed

+540
-5
lines changed

‎modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
import org.apache.ignite.plugin.CachePluginConfiguration;
6464
import org.apache.ignite.spi.encryption.EncryptionSpi;
6565
import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi;
66+
import org.apache.ignite.topology.MdcTopologyValidator;
6667
import org.jetbrains.annotations.Nullable;
6768

6869
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DEFAULT_DISK_PAGE_COMPRESSION;
@@ -2221,6 +2222,14 @@ public TopologyValidator getTopologyValidator() {
22212222
* @return {@code this} for chaining.
22222223
*/
22232224
public CacheConfiguration<K, V> setTopologyValidator(TopologyValidator topValidator) {
2225+
try {
2226+
if (topValidator instanceof MdcTopologyValidator)
2227+
((MdcTopologyValidator)topValidator).checkConfiguration();
2228+
}
2229+
catch (Exception e) {
2230+
throw new CacheException(e);
2231+
}
2232+
22242233
this.topValidator = topValidator;
22252234

22262235
return this;

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -409,8 +409,11 @@ private void checkCache(CacheJoinNodeDiscoveryData.CacheInfo locInfo, CacheData
409409
CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "cachePreloadMode",
410410
"Cache preload mode", locAttr.cacheRebalanceMode(), rmtAttr.cacheRebalanceMode(), true);
411411

412-
CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "topologyValidator",
413-
"Cache topology validator", locAttr.topologyValidatorClassName(), rmtAttr.topologyValidatorClassName(), true);
412+
CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "topologyValidatorClass",
413+
"Cache topology validator class", locAttr.topologyValidatorClassName(), rmtAttr.topologyValidatorClassName(), true);
414+
415+
CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "topologyValidator", "Cache topology validator",
416+
locAttr.configuration().getTopologyValidator(), rmtAttr.configuration().getTopologyValidator(), true);
414417

415418
ClusterNode rmtNode = ctx.discovery().node(rmt);
416419

@@ -2576,9 +2579,12 @@ private void validateCacheGroupConfiguration(CacheConfiguration cfg, CacheConfig
25762579
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg, "dataRegionName", "Data region",
25772580
cfg.getDataRegionName(), startCfg.getDataRegionName(), true);
25782581

2579-
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg, "topologyValidator", "Topology validator",
2582+
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg, "topologyValidatorClass", "Topology validator class",
25802583
attr1.topologyValidatorClassName(), attr2.topologyValidatorClassName(), true);
25812584

2585+
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg, "topologyValidator", "Topology validator",
2586+
cfg.getTopologyValidator(), startCfg.getTopologyValidator(), true);
2587+
25822588
CU.validateCacheGroupsAttributesMismatch(log, cfg, startCfg, "partitionLossPolicy", "Partition Loss Policy",
25832589
cfg.getPartitionLossPolicy(), startCfg.getPartitionLossPolicy(), true);
25842590

‎modules/core/src/main/java/org/apache/ignite/topology/MdcTopologyValidator.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.ignite.topology;
19+
20+
import java.util.Collection;
21+
import java.util.Objects;
22+
import java.util.Set;
23+
import java.util.stream.Stream;
24+
import org.apache.ignite.IgniteSystemProperties;
25+
import org.apache.ignite.cluster.ClusterNode;
26+
import org.apache.ignite.configuration.TopologyValidator;
27+
import org.apache.ignite.lang.IgniteExperimental;
28+
29+
/**
30+
* Multi-Datacenter Topology Validator.
31+
*
32+
* <p>This class is used to validate the cluster topology in a multi-datacenter (MDC) environment
33+
* by enforcing rules based on the visibility of datacenters.
34+
* It provides protection against split-brain scenarios during datacenter failures or unavailability due to network issues.</p>
35+
*
36+
* <p>
37+
* In order to use MdcTopologyValidator one has to specify datacenter ID attribute on each server node.
38+
* Datacenter ID is an arbitrary string that can be set with {@link IgniteSystemProperties#IGNITE_DATA_CENTER_ID}
39+
* system property on the node startup.
40+
* All server nodes belonging to the same datacenter should specify the same datacenter ID, and nodes in different datacenters
41+
* should have different datacenter IDs.
42+
* </p>
43+
* <p>The validator supports two modes of operation:</p>
44+
* <ul>
45+
* <li><strong>Majority-based validation:</strong> When an odd number of datacenters are defined, the validator enables
46+
* data modification operations in the cluster segment that contain a majority of datacenters.
47+
* Any segment containing a minority of datacenters is considered as invalid with only read operations available.</li>
48+
* <li><strong>Main Datacenter validation:</strong> When an even number of datacenters are defined, a main datacenter
49+
* should be specified. The cluster segment remains write-accessible as long as the main datacenter is visible from
50+
* all nodes of that segment.</li>
51+
* </ul>
52+
*
53+
* <p><strong>Usage Requirements:</strong></p>
54+
* <ul>
55+
* <li>If number of datacenters is even, specify a main datacenter via {@link #setMainDatacenter(String)}.
56+
* Set of datacenters could be left null.</li>
57+
* <li>If number of datacenters is odd, set of datacenter IDs must be specified via {@link #setDatacenters(Set)}.
58+
* Main datacenter setting is ignored and could be left null.</li>
59+
*
60+
* </ul>
61+
*
62+
* <p><strong>Example:</strong></p>
63+
* <pre>
64+
* MdcTopologyValidator mdcValidator = new MdcTopologyValidator();
65+
* mdcValidator.setDatacenters(Set.of("DC1", "DC2", "DC3"));
66+
*
67+
* CacheConfiguration cacheCfg = new CacheConfiguration("example-cache")
68+
* .setTopologyValidator(mdcValidator)
69+
* // other cache properties.
70+
* </pre>
71+
*
72+
* <p><strong>Note:</strong> This class is marked with the {@link IgniteExperimental} annotation and may change in future releases.</p>
73+
*
74+
* @see TopologyValidator
75+
* @since Apache Ignite 2.18
76+
*/
77+
@IgniteExperimental
78+
public class MdcTopologyValidator implements TopologyValidator {
79+
/** */
80+
private static final long serialVersionUID = 0L;
81+
82+
/** */
83+
private Set<String> dcs;
84+
85+
/** */
86+
private String mainDc;
87+
88+
/** @param datacenters Datacenters.*/
89+
public void setDatacenters(Set<String> datacenters) {
90+
dcs = datacenters;
91+
}
92+
93+
/** @param mainDatacenter Main datacenter.*/
94+
public void setMainDatacenter(String mainDatacenter) {
95+
mainDc = mainDatacenter;
96+
}
97+
98+
/** */
99+
public void checkConfiguration() {
100+
if (dcs == null && mainDc == null)
101+
throw new IllegalStateException("Either set of datacenters or main datacenter should be specified.");
102+
103+
if (dcs != null && dcs.isEmpty())
104+
throw new IllegalStateException("Please provide a non-empty set of datacenters.");
105+
106+
if (mainDc != null && dcs != null && dcs.size() % 2 == 1)
107+
throw new IllegalStateException("Uneven number of datacenters cannot be used along with main datacenter. " +
108+
"Please remove main datacenter setting or specify even number of datacenters.");
109+
}
110+
111+
/** {@inheritDoc} */
112+
@Override public boolean validate(Collection<ClusterNode> nodes) {
113+
Stream<ClusterNode> servers = nodes.stream().filter(node -> !node.isClient());
114+
115+
if (mainDc != null)
116+
return servers.anyMatch(n -> n.dataCenterId() != null && n.dataCenterId().equals(mainDc));
117+
118+
long visible = servers.map(ClusterNode::dataCenterId).distinct().count();
119+
int half = dcs.size() / 2;
120+
121+
return visible > half;
122+
}
123+
124+
/** {@inheritDoc} */
125+
@Override public boolean equals(Object o) {
126+
if (o == null || getClass() != o.getClass())
127+
return false;
128+
129+
MdcTopologyValidator validator = (MdcTopologyValidator)o;
130+
131+
return Objects.equals(dcs, validator.dcs) && Objects.equals(mainDc, validator.mainDc);
132+
}
133+
134+
/** {@inheritDoc} */
135+
@Override public int hashCode() {
136+
return Objects.hash(dcs, mainDc);
137+
}
138+
}

‎modules/core/src/main/java/org/apache/ignite/topology/package-info.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
/**
19+
* <!-- Package description. -->
20+
* Contains topology-related classes.
21+
*/
22+
23+
package org.apache.ignite.topology;

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements
6262
.setCacheMode(CacheMode.REPLICATED)
6363
.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
6464
.setTopologyValidator(new TopologyValidator() {
65+
@Override public boolean equals(Object obj) {
66+
return true;
67+
}
68+
6569
@Override public boolean validate(Collection<ClusterNode> nodes) {
6670
return nodes.size() == 2;
6771
}

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,13 +335,21 @@ private static class TestCacheTopologyValidatorProvider implements CacheTopology
335335
@Override public TopologyValidator topologyValidator(String cacheName) {
336336
if (CACHE_NAME_1.equals(cacheName)) {
337337
return new TopologyValidator() {
338+
@Override public boolean equals(Object obj) {
339+
return true;
340+
}
341+
338342
@Override public boolean validate(Collection<ClusterNode> nodes) {
339343
return servers(nodes) == 2;
340344
}
341345
};
342346
}
343347
else if (CACHE_NAME_2.equals(cacheName)) {
344348
return new TopologyValidator() {
349+
@Override public boolean equals(Object obj) {
350+
return true;
351+
}
352+
345353
@Override public boolean validate(Collection<ClusterNode> nodes) {
346354
return servers(nodes) >= 2;
347355
}

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorCacheGroupsAbstractTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,21 @@ private static class TestCacheGroupTopologyValidatorProvider implements CacheTop
134134
@Override public TopologyValidator topologyValidator(String grpName) {
135135
if (GROUP_1.equals(grpName)) {
136136
return new TopologyValidator() {
137+
@Override public boolean equals(Object obj) {
138+
return true;
139+
}
140+
137141
@Override public boolean validate(Collection<ClusterNode> nodes) {
138142
return nodes.size() == 2;
139143
}
140144
};
141145
}
142146
else if (GROUP_2.equals(grpName)) {
143147
return new TopologyValidator() {
148+
@Override public boolean equals(Object obj) {
149+
return true;
150+
}
151+
144152
@Override public boolean validate(Collection<ClusterNode> nodes) {
145153
return nodes.size() >= 2;
146154
}

‎modules/core/src/test/java/org/apache/ignite/spi/discovery/datacenter/MultiDataCenterDeploymentTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ public class MultiDataCenterDeploymentTest extends GridCommonAbstractTest {
5454
super.afterTest();
5555

5656
stopAllGrids();
57+
58+
System.clearProperty(IgniteSystemProperties.IGNITE_DATA_CENTER_ID);
5759
}
5860

5961
/**

0 commit comments

Comments
 (0)