Skip to content

Commit 31de9c2

Browse files
astefanastefan
authored
Merge branch 'main' into inlinestats_snapshot_only
2 parents 06f84b8 + 2c846e7 commit 31de9c2

File tree

6 files changed

+121
-10
lines changed

6 files changed

+121
-10
lines changed

‎docs/changelog/122365.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122365
2+
summary: Fix handling of auto expand replicas for stateless indices
3+
area: "Search"
4+
type: bug
5+
issues: []

‎server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,9 +156,8 @@ public static Map<Integer, List<String>> getAutoExpandReplicaChanges(
156156
)) {
157157
if (indexMetadata.getNumberOfReplicas() == 0) {
158158
nrReplicasChanged.computeIfAbsent(1, ArrayList::new).add(indexMetadata.getIndex().getName());
159-
} else {
160-
continue;
161159
}
160+
continue;
162161
}
163162
if (allocation == null) {
164163
allocation = allocationSupplier.get();

‎server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
2121
import org.elasticsearch.cluster.routing.RoutingNodesHelper;
2222
import org.elasticsearch.cluster.routing.ShardRoutingState;
23+
import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
2324
import org.elasticsearch.common.settings.Settings;
2425
import org.elasticsearch.core.Strings;
26+
import org.elasticsearch.index.IndexVersion;
2527
import org.elasticsearch.indices.cluster.ClusterStateChanges;
2628
import org.elasticsearch.test.ESTestCase;
2729
import org.elasticsearch.threadpool.TestThreadPool;
@@ -31,11 +33,14 @@
3133
import java.util.Collections;
3234
import java.util.HashSet;
3335
import java.util.List;
36+
import java.util.Map;
3437
import java.util.Set;
3538
import java.util.concurrent.atomic.AtomicInteger;
3639
import java.util.stream.Collectors;
3740

41+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING;
3842
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS;
43+
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
3944
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
4045
import static org.hamcrest.Matchers.equalTo;
4146
import static org.hamcrest.Matchers.everyItem;
@@ -221,4 +226,48 @@ public void testCalculateDesiredNumberOfReplicas() {
221226
assertThat(autoExpandReplicas.calculateDesiredNumberOfReplicas(matchingNodes), equalTo(Math.max(lowerBound, matchingNodes - 1)));
222227
assertThat(autoExpandReplicas.calculateDesiredNumberOfReplicas(max + 1), equalTo(max));
223228
}
229+
230+
public void testGetAutoExpandReplicaChangesStatelessIndices() {
231+
{
232+
// number of replicas is adjusted to 1 when it is initialized to 0
233+
Metadata metadata = Metadata.builder()
234+
.put(
235+
IndexMetadata.builder("test")
236+
.settings(
237+
Settings.builder()
238+
.put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "stateless")
239+
.put("index.version.created", IndexVersion.current())
240+
.put(SETTING_NUMBER_OF_SHARDS, 1)
241+
.put(SETTING_NUMBER_OF_REPLICAS, 0)
242+
.put(INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-all")
243+
)
244+
)
245+
.build();
246+
Map<Integer, List<String>> autoExpandReplicaChanges = AutoExpandReplicas.getAutoExpandReplicaChanges(metadata, null);
247+
assertEquals(1, autoExpandReplicaChanges.size());
248+
List<String> indices = autoExpandReplicaChanges.get(1);
249+
assertEquals(1, indices.size());
250+
assertEquals("test", indices.getFirst());
251+
}
252+
{
253+
// no changes when number of replicas is set to anything other than 0
254+
Metadata metadata = Metadata.builder()
255+
.put(
256+
IndexMetadata.builder("test")
257+
.settings(
258+
Settings.builder()
259+
.put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "stateless")
260+
.put("index.version.created", IndexVersion.current())
261+
.put(SETTING_NUMBER_OF_SHARDS, 1)
262+
.put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(1, 10))
263+
.put(INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-all")
264+
)
265+
)
266+
.build();
267+
Map<Integer, List<String>> autoExpandReplicaChanges = AutoExpandReplicas.getAutoExpandReplicaChanges(metadata, () -> {
268+
throw new UnsupportedOperationException();
269+
});
270+
assertEquals(0, autoExpandReplicaChanges.size());
271+
}
272+
}
224273
}

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@
3838

3939
import static org.elasticsearch.test.MapMatcher.assertMap;
4040
import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME;
41-
import static org.hamcrest.Matchers.*;
41+
import static org.hamcrest.Matchers.any;
42+
import static org.hamcrest.Matchers.equalTo;
43+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
44+
import static org.hamcrest.Matchers.hasKey;
4245

4346
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
4447
public class MultiClustersIT extends ESRestTestCase {

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@
1414
import org.elasticsearch.client.ResponseException;
1515
import org.elasticsearch.client.RestClient;
1616
import org.elasticsearch.core.IOUtils;
17+
import org.elasticsearch.test.MapMatcher;
1718
import org.elasticsearch.test.TestClustersThreadFilter;
1819
import org.elasticsearch.test.cluster.ElasticsearchCluster;
1920
import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase;
21+
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
22+
import org.hamcrest.Matcher;
2023
import org.junit.After;
2124
import org.junit.AfterClass;
2225
import org.junit.Before;
@@ -25,6 +28,12 @@
2528
import org.junit.rules.TestRule;
2629

2730
import java.io.IOException;
31+
import java.util.Map;
32+
33+
import static org.elasticsearch.test.MapMatcher.assertMap;
34+
import static org.elasticsearch.test.MapMatcher.matchesMap;
35+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
36+
import static org.hamcrest.Matchers.instanceOf;
2837

2938
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
3039
public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase {
@@ -49,6 +58,8 @@ public void setRemoteClient() throws IOException {
4958
}
5059
}
5160

61+
private boolean isCCSRequest;
62+
5263
@AfterClass
5364
public static void closeRemoteClients() throws IOException {
5465
try {
@@ -66,13 +77,20 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin
6677

6778
@Override
6879
protected String from(String... indexName) {
69-
if (randomBoolean()) {
80+
isCCSRequest = randomBoolean();
81+
if (isCCSRequest) {
7082
return "FROM *:" + String.join(",*:", indexName);
7183
} else {
7284
return "FROM " + String.join(",", indexName);
7385
}
7486
}
7587

88+
@Override
89+
public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException {
90+
requestObject.includeCCSMetadata(true);
91+
return super.runEsql(requestObject);
92+
}
93+
7694
@After
7795
public void wipeRemoteTestData() throws IOException {
7896
try {
@@ -82,4 +100,35 @@ public void wipeRemoteTestData() throws IOException {
82100
assertEquals(404, re.getResponse().getStatusLine().getStatusCode());
83101
}
84102
}
103+
104+
private MapMatcher getClustersMetadataMatcher() {
105+
MapMatcher mapMatcher = matchesMap();
106+
mapMatcher = mapMatcher.entry("running", 0);
107+
mapMatcher = mapMatcher.entry("total", 1);
108+
mapMatcher = mapMatcher.entry("failed", 0);
109+
mapMatcher = mapMatcher.entry("partial", 0);
110+
mapMatcher = mapMatcher.entry("successful", 1);
111+
mapMatcher = mapMatcher.entry("skipped", 0);
112+
mapMatcher = mapMatcher.entry(
113+
"details",
114+
matchesMap().entry(
115+
Clusters.REMOTE_CLUSTER_NAME,
116+
matchesMap().entry("_shards", matchesMap().extraOk())
117+
.entry("took", greaterThanOrEqualTo(0))
118+
.entry("indices", instanceOf(String.class))
119+
.entry("status", "successful")
120+
)
121+
);
122+
return mapMatcher;
123+
}
124+
125+
@Override
126+
protected void assertQueryResult(Map<String, Object> result, Matcher<?> columnMatcher, Matcher<?> valuesMatcher) {
127+
var matcher = getResultMatcher(result).entry("columns", columnMatcher).entry("values", valuesMatcher);
128+
if (isCCSRequest) {
129+
matcher = matcher.entry("_clusters", getClustersMetadataMatcher());
130+
}
131+
assertMap(result, matcher);
132+
}
133+
85134
}

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.xcontent.XContentType;
1818
import org.elasticsearch.xpack.esql.AssertWarnings;
1919
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
20+
import org.hamcrest.Matcher;
2021
import org.junit.After;
2122
import org.junit.Assert;
2223

@@ -62,7 +63,7 @@ public void testTimestampFilterFromQuery() throws IOException {
6263

6364
// filter includes both indices in the result (all columns, all rows)
6465
RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*"));
65-
assertResultMap(
66+
assertQueryResult(
6667
runEsql(builder),
6768
matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
6869
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
@@ -73,7 +74,7 @@ public void testTimestampFilterFromQuery() throws IOException {
7374

7475
// filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
7576
builder = timestampFilter("gte", "2024-01-01").query(from("test*"));
76-
assertResultMap(
77+
assertQueryResult(
7778
runEsql(builder),
7879
matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
7980
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
@@ -84,7 +85,7 @@ public void testTimestampFilterFromQuery() throws IOException {
8485
// filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds
8586
// after eliminating the index filter. All columns are returned.
8687
builder = timestampFilter("gte", "2025-01-01").query(from("test*"));
87-
assertResultMap(
88+
assertQueryResult(
8889
runEsql(builder),
8990
matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
9091
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
@@ -102,7 +103,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
102103

103104
// filter includes only test1. Columns and rows of test2 are filtered out
104105
RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*"));
105-
assertResultMap(
106+
assertQueryResult(
106107
runEsql(builder),
107108
matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
108109
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
@@ -113,7 +114,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
113114
// filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
114115
builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*");
115116
Map<String, Object> result = runEsql(builder);
116-
assertResultMap(
117+
assertQueryResult(
117118
result,
118119
matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword"))
119120
.item(matchesMap().entry("name", "id1").entry("type", "integer")),
@@ -138,7 +139,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw
138139
from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*"
139140
);
140141
Map<String, Object> result = runEsql(builder);
141-
assertResultMap(
142+
assertQueryResult(
142143
result,
143144
matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword"))
144145
.item(matchesMap().entry("name", "id1").entry("type", "integer"))
@@ -298,4 +299,9 @@ protected void indexTimestampDataForClient(RestClient client, int docs, String i
298299
Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
299300
}
300301
}
302+
303+
protected void assertQueryResult(Map<String, Object> result, Matcher<?> columnMatcher, Matcher<?> valuesMatcher) {
304+
assertResultMap(result, columnMatcher, valuesMatcher);
305+
}
306+
301307
}

0 commit comments

Comments
 (0)