Skip to content

Index Balanced Allocation Decider Add index routing filter handling #138741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+74 −13
Open

Index Balanced Allocation Decider Add index routing filter handling #138741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8a9a4f8
Add index routing filter handling
zhubotang-wq Nov 28, 2025
35f41eb
Update server/src/main/java/org/elasticsearch/cluster/routing/allocat…
zhubotang-wq Nov 28, 2025
1d01c0c
Merge branch 'main' into 12080-index-shard-count-allocation-decider-f…
zhubotang-wq Nov 28, 2025
fd6ff2f
Add index routing filter handling
zhubotang-wq Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion ...a/org/elasticsearch/cluster/routing/allocation/decider/IndexBalanceAllocationDecider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
}

final ProjectId projectId = allocation.getClusterState().metadata().projectFor(index).id();
final IndexMetadata indexMetadata = allocation.getClusterState().metadata().getProject(projectId).index(index);

if (hasIndexRoutingFilters(indexMetadata)) {
return allocation.decision(Decision.YES, NAME, "Decider is disabled for index level allocation filters.");
}

final Set<DiscoveryNode> eligibleNodes = new HashSet<>();
int totalShards = 0;
String nomenclature = EMPTY;
Expand All @@ -104,7 +110,6 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
} else if (node.node().getRoles().contains(SEARCH_ROLE)) {
collectEligibleNodes(allocation, eligibleNodes, SEARCH_ROLE);
// Replicas only.
final IndexMetadata indexMetadata = allocation.getClusterState().metadata().getProject(projectId).index(index);
totalShards = indexMetadata.getNumberOfShards() * indexMetadata.getNumberOfReplicas();
nomenclature = "search";
}
Expand Down Expand Up @@ -169,4 +174,10 @@ private boolean hasFilters() {
|| (clusterIncludeFilters != null && clusterIncludeFilters.hasFilters())
|| (clusterRequireFilters != null && clusterRequireFilters.hasFilters());
}

private boolean hasIndexRoutingFilters(IndexMetadata indexMetadata) {
return indexMetadata.requireFilters() != null && indexMetadata.requireFilters().hasFilters()
|| indexMetadata.excludeFilters() != null && indexMetadata.excludeFilters().hasFilters()
|| indexMetadata.includeFilters() != null && indexMetadata.includeFilters().hasFilters();
}
}
74 changes: 62 additions & 12 deletions .../elasticsearch/cluster/routing/allocation/decider/IndexBalanceAllocationDeciderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX;
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX;
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE;
import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder;
import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX;
Expand Down Expand Up @@ -77,12 +81,12 @@ public class IndexBalanceAllocationDeciderTests extends ESAllocationTestCase {
private List<RoutingNode> indexTier;
private List<RoutingNode> searchIier;

private void setup(Settings settings) {
private void setup(Settings clusterSettings, Supplier<Settings> indexSettings) {
Copy link
Contributor Author

@zhubotang-wq zhubotang-wq Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason why indexSettings is a Supplier is due to the need to defer retrieving DiscoveryNode reference until these variables are initialized.

String id = randomFrom(indexNodeOne.getId(), indexNodeTwo.getId(), searchNodeOne.getId(), searchNodeTwo.getId());
String hostName = randomFrom(
indexNodeOne.getHostName(),
indexNodeTwo.getHostName(),
searchNodeOne.getHostName(),
searchNodeTwo.getHostName()
);

Copy link
Member

@ywangd ywangd Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's OK though the filters and the decider behaviour do not depend on the filtered node IDs to match existing nodes.

final String indexName = "IndexBalanceAllocationDeciderIndex";
final Map<DiscoveryNode, List<ShardRouting>> nodeToShardRoutings = new HashMap<>();

Settings.Builder builder = Settings.builder()
.put(settings)
.put(clusterSettings)
.put("stateless.enabled", "true")
.put(IndexBalanceConstraintSettings.INDEX_BALANCE_DECIDER_ENABLED_SETTING.getKey(), "true");

Expand Down Expand Up @@ -111,10 +115,9 @@ private void setup(Settings settings) {

indexMetadata = IndexMetadata.builder(indexName)
.settings(
indexSettings(IndexVersion.current(), numberOfPrimaryShards, replicationFactor).put(
SETTING_CREATION_DATE,
System.currentTimeMillis()
).build()
indexSettings(IndexVersion.current(), numberOfPrimaryShards, replicationFactor).put(indexSettings.get())
.put(SETTING_CREATION_DATE, System.currentTimeMillis())
.build()
)
.timestampRange(IndexLongFieldRange.UNKNOWN)
.eventIngestedRange(IndexLongFieldRange.UNKNOWN)
Expand Down Expand Up @@ -211,8 +214,8 @@ private void setup(Settings settings) {
}

public void testCanAllocateUnderThresholdWithExcessShards() {
Settings settings = allowExcessShards(Settings.EMPTY);
setup(settings);
Settings clusterSettings = allowExcessShards(Settings.EMPTY);
setup(clusterSettings, () -> Settings.EMPTY);

ShardRouting newIndexShardRouting = TestShardRouting.newShardRouting(
new ShardId("newIndex", "uuid", 1),
Expand Down Expand Up @@ -279,7 +282,7 @@ private void verifyCanAllocate() {
}

public void testCanAllocateExceedThreshold() {
setup(Settings.EMPTY);
setup(Settings.EMPTY, () -> Settings.EMPTY);

int ideal = numberOfPrimaryShards / 2;
int current = numberOfPrimaryShards / 2;
Expand Down Expand Up @@ -316,11 +319,11 @@ public void testCanAllocateExceedThreshold() {
}

public void testCanAllocateHasDiscoveryNodeFilters() {
Settings settings = addRandomFilterSetting(Settings.EMPTY);
Settings clusterSettings = addRandomFilterSetting(Settings.EMPTY);
if (randomBoolean()) {
settings = allowExcessShards(settings);
clusterSettings = allowExcessShards(clusterSettings);
}
setup(settings);
setup(clusterSettings, () -> Settings.EMPTY);

for (RoutingNode routingNode : indexTier) {
assertDecisionMatches(
Expand All @@ -341,6 +344,28 @@ public void testCanAllocateHasDiscoveryNodeFilters() {
}
}

public void testCanAllocateHasIndexRoutingFilters() {
setup(Settings.EMPTY, this::addRandomIndexRoutingFilters);

for (RoutingNode routingNode : indexTier) {
assertDecisionMatches(
"Having DiscoveryNodeFilters disables this decider",
indexBalanceAllocationDecider.canAllocate(indexTierShardRouting, routingNode, routingAllocation),
Decision.Type.YES,
"Decider is disabled for index level allocation filters."
);
}

for (RoutingNode routingNode : searchIier) {
assertDecisionMatches(
"Having DiscoveryNodeFilters disables this decider",
indexBalanceAllocationDecider.canAllocate(searchTierShardRouting, routingNode, routingAllocation),
Decision.Type.YES,
"Decider is disabled for index level allocation filters."
);
}
}

public Settings addRandomFilterSetting(Settings settings) {
String setting = randomFrom(
CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX,
Expand All @@ -362,4 +387,29 @@ public Settings allowExcessShards(Settings settings) {
.build();
}

public Settings addRandomIndexRoutingFilters() {
String setting = randomFrom(
INDEX_ROUTING_REQUIRE_GROUP_PREFIX,
INDEX_ROUTING_INCLUDE_GROUP_PREFIX,
INDEX_ROUTING_EXCLUDE_GROUP_PREFIX
);
String attribute = randomFrom("_ip", "_host", "_id");
String ip = randomFrom("192.168.0.1", "192.168.0.2", "192.168.7.1", "10.17.0.1");
String id = randomFrom(indexNodeOne.getId(), indexNodeTwo.getId(), searchNodeOne.getId(), searchNodeTwo.getId());
String hostName = randomFrom(
indexNodeOne.getHostName(),
indexNodeTwo.getHostName(),
searchNodeOne.getHostName(),
searchNodeTwo.getHostName()
);

String value = switch (attribute) {
case "_ip" -> ip;
case "_host" -> hostName;
case "_id" -> id;
default -> throw new IllegalStateException("Unexpected value: " + attribute);
};
return Settings.builder().put(setting + "." + attribute, value).build();
}

}