Skip to content

Commit 6e8521a

Browse files
kderussokderusso
committed
Merge branch 'main' into kderusso/rerank-snippet-poc
2 parents 44c2c71 + 57c5313 commit 6e8521a

File tree

22 files changed

+548
-59
lines changed

22 files changed

+548
-59
lines changed

‎docs/changelog/131677.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131677
2+
summary: Throw better exception if verifying empty repo
3+
area: Snapshot/Restore
4+
type: bug
5+
issues: []

‎muted-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -493,6 +493,12 @@ tests:
493493
- class: org.elasticsearch.compute.lucene.read.SortedSetOrdinalsBuilderTests
494494
method: testReader
495495
issue: https://github.com/elastic/elasticsearch/issues/131573
496+
- class: org.elasticsearch.indices.cluster.RemoteSearchForceConnectTimeoutIT
497+
method: testTimeoutSetting
498+
issue: https://github.com/elastic/elasticsearch/issues/131656
499+
- class: org.elasticsearch.search.SearchWithIndexBlocksIT
500+
method: testSearchShardsOnIndicesWithIndexRefreshBlocks
501+
issue: https://github.com/elastic/elasticsearch/issues/131662
496502

497503
# Examples:
498504
#

‎rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_custom.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"inference.put_custom": {
3+
"documentation": {
4+
"url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put-custom",
5+
"description": "Configure a custom inference endpoint"
6+
},
7+
"stability": "stable",
8+
"visibility": "public",
9+
"headers": {
10+
"accept": ["application/json"],
11+
"content_type": ["application/json"]
12+
},
13+
"url": {
14+
"paths": [
15+
{
16+
"path": "/_inference/{task_type}/{custom_inference_id}",
17+
"methods": ["PUT"],
18+
"parts": {
19+
"task_type": {
20+
"type": "string",
21+
"description": "The task type"
22+
},
23+
"custom_inference_id": {
24+
"type": "string",
25+
"description": "The inference Id"
26+
}
27+
}
28+
}
29+
]
30+
},
31+
"body": {
32+
"description": "The inference endpoint's task and service settings"
33+
}
34+
}
35+
}

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mtermvectors/30_routing.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@ routing:
77
settings:
88
index:
99
number_of_shards: 5
10-
number_of_replicas: 0
1110

1211
- do:
1312
cluster.health:
14-
wait_for_status: green
13+
wait_for_no_initializing_shards: true
1514

1615
- do:
1716
index:
@@ -52,11 +51,14 @@ requires routing:
5251
settings:
5352
index:
5453
number_of_shards: 5
55-
number_of_replicas: 0
5654
mappings:
5755
_routing:
5856
required: true
5957

58+
- do:
59+
cluster.health:
60+
wait_for_no_initializing_shards: true
61+
6062
- do:
6163
index:
6264
index: test_1

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/termvectors/20_issue7121.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
index:
88
translog.flush_threshold_size: "512MB"
99
number_of_shards: 1
10-
number_of_replicas: 0
1110
refresh_interval: -1
1211
mappings:
1312
properties:
@@ -17,7 +16,7 @@
1716

1817
- do:
1918
cluster.health:
20-
wait_for_status: green
19+
wait_for_no_initializing_shards: true
2120

2221
- do:
2322
index:

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/termvectors/30_realtime.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@
77
settings:
88
index:
99
refresh_interval: -1
10-
number_of_replicas: 0
1110

1211
- do:
13-
cluster.health:
14-
wait_for_status: green
12+
cluster.health:
13+
wait_for_no_initializing_shards: true
1514

1615
- do:
1716
index:

‎server/src/internalClusterTest/java/org/elasticsearch/index/engine/ThreadPoolMergeSchedulerStressTestIT.java‎

Lines changed: 7 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
1717
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
1818
import org.elasticsearch.action.admin.indices.segments.ShardSegments;
19+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
1920
import org.elasticsearch.common.settings.Settings;
2021
import org.elasticsearch.common.util.CollectionUtils;
2122
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -44,6 +45,7 @@
4445
import java.util.concurrent.atomic.AtomicReference;
4546

4647
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
48+
import static org.hamcrest.Matchers.equalTo;
4749
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
4850
import static org.hamcrest.Matchers.instanceOf;
4951
import static org.hamcrest.Matchers.is;
@@ -272,12 +274,11 @@ public void testMergingFallsBehindAndThenCatchesUp() throws Exception {
272274
assertThat(testEnginePlugin.enqueuedMergesSet.size(), is(0));
273275
testEnginePlugin.mergeExecutorServiceReference.get().allDone();
274276
}, 1, TimeUnit.MINUTES);
275-
var segmentsCountAfterMergingCaughtUp = getSegmentsCountForAllShards("index");
276-
// force merge should be a noop after all available merging was done
277-
assertAllSuccessful(indicesAdmin().prepareForceMerge("index").get());
278-
var segmentsCountAfterForceMerge = getSegmentsCountForAllShards("index");
279-
assertThat(segmentsCountAfterForceMerge, is(segmentsCountAfterMergingCaughtUp));
280-
// let's also run a force-merge to 1 segment
277+
// indices stats says that no merge is currently running (meaning merging did catch up)
278+
IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats("index").setMerge(true).get();
279+
long currentMergeCount = indicesStatsResponse.getIndices().get("index").getPrimaries().merge.getCurrent();
280+
assertThat(currentMergeCount, equalTo(0L));
281+
// run a force-merge to 1 segment to make sure nothing is broken
281282
assertAllSuccessful(indicesAdmin().prepareForceMerge("index").setMaxNumSegments(1).get());
282283
assertAllSuccessful(indicesAdmin().prepareRefresh("index").get());
283284
// assert one segment per shard
@@ -292,20 +293,6 @@ public void testMergingFallsBehindAndThenCatchesUp() throws Exception {
292293
}
293294
}
294295

295-
private int getSegmentsCountForAllShards(String indexName) {
296-
// refresh, otherwise we'd be still seeing the old merged-away segments
297-
assertAllSuccessful(indicesAdmin().prepareRefresh(indexName).get());
298-
int count = 0;
299-
IndicesSegmentResponse indicesSegmentResponse = indicesAdmin().prepareSegments(indexName).get();
300-
Iterator<IndexShardSegments> indexShardSegmentsIterator = indicesSegmentResponse.getIndices().get(indexName).iterator();
301-
while (indexShardSegmentsIterator.hasNext()) {
302-
for (ShardSegments segments : indexShardSegmentsIterator.next()) {
303-
count += segments.getSegments().size();
304-
}
305-
}
306-
return count;
307-
}
308-
309296
private TestEnginePlugin getTestEnginePlugin() {
310297
return getInstanceFromNode(PluginsService.class).filterPlugins(TestEnginePlugin.class).toList().get(0);
311298
}

‎server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithIndexBlocksIT.java‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.search;
11+
12+
import org.elasticsearch.action.index.IndexRequestBuilder;
13+
import org.elasticsearch.action.search.ClosePointInTimeRequest;
14+
import org.elasticsearch.action.search.OpenPointInTimeRequest;
15+
import org.elasticsearch.action.search.SearchRequest;
16+
import org.elasticsearch.action.search.SearchShardsGroup;
17+
import org.elasticsearch.action.search.SearchShardsRequest;
18+
import org.elasticsearch.action.search.TransportClosePointInTimeAction;
19+
import org.elasticsearch.action.search.TransportOpenPointInTimeAction;
20+
import org.elasticsearch.action.search.TransportSearchShardsAction;
21+
import org.elasticsearch.action.support.IndicesOptions;
22+
import org.elasticsearch.cluster.ClusterState;
23+
import org.elasticsearch.cluster.block.ClusterBlocks;
24+
import org.elasticsearch.cluster.metadata.IndexMetadata;
25+
import org.elasticsearch.cluster.metadata.ProjectId;
26+
import org.elasticsearch.cluster.node.DiscoveryNode;
27+
import org.elasticsearch.cluster.service.ClusterService;
28+
import org.elasticsearch.common.bytes.BytesReference;
29+
import org.elasticsearch.core.TimeValue;
30+
import org.elasticsearch.index.query.MatchAllQueryBuilder;
31+
import org.elasticsearch.index.query.QueryBuilders;
32+
import org.elasticsearch.search.builder.PointInTimeBuilder;
33+
import org.elasticsearch.search.builder.SearchSourceBuilder;
34+
import org.elasticsearch.test.ESIntegTestCase;
35+
36+
import java.util.ArrayList;
37+
import java.util.HashMap;
38+
import java.util.List;
39+
import java.util.Map;
40+
import java.util.Objects;
41+
42+
import static org.elasticsearch.cluster.block.ClusterBlocks.EMPTY_CLUSTER_BLOCK;
43+
import static org.elasticsearch.test.ClusterServiceUtils.setState;
44+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
45+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
46+
47+
public class SearchWithIndexBlocksIT extends ESIntegTestCase {
48+
49+
public void testSearchIndicesWithIndexRefreshBlocks() {
50+
List<String> indices = createIndices();
51+
Map<String, Integer> numDocsPerIndex = indexDocuments(indices);
52+
List<String> unblockedIndices = addIndexRefreshBlockToSomeIndices(indices);
53+
54+
int expectedHits = 0;
55+
for (String index : unblockedIndices) {
56+
expectedHits += numDocsPerIndex.get(index);
57+
}
58+
59+
assertHitCount(prepareSearch().setQuery(QueryBuilders.matchAllQuery()), expectedHits);
60+
}
61+
62+
public void testOpenPITOnIndicesWithIndexRefreshBlocks() {
63+
List<String> indices = createIndices();
64+
Map<String, Integer> numDocsPerIndex = indexDocuments(indices);
65+
List<String> unblockedIndices = addIndexRefreshBlockToSomeIndices(indices);
66+
67+
int expectedHits = 0;
68+
for (String index : unblockedIndices) {
69+
expectedHits += numDocsPerIndex.get(index);
70+
}
71+
72+
BytesReference pitId = null;
73+
try {
74+
OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest(indices.toArray(new String[0])).keepAlive(
75+
TimeValue.timeValueSeconds(10)
76+
).allowPartialSearchResults(true);
77+
pitId = client().execute(TransportOpenPointInTimeAction.TYPE, openPITRequest).actionGet().getPointInTimeId();
78+
SearchRequest searchRequest = new SearchRequest().source(
79+
new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(pitId).setKeepAlive(TimeValue.timeValueSeconds(10)))
80+
);
81+
assertHitCount(client().search(searchRequest), expectedHits);
82+
} finally {
83+
if (pitId != null) {
84+
client().execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pitId)).actionGet();
85+
}
86+
}
87+
}
88+
89+
public void testMultiSearchIndicesWithIndexRefreshBlocks() {
90+
List<String> indices = createIndices();
91+
Map<String, Integer> numDocsPerIndex = indexDocuments(indices);
92+
List<String> unblockedIndices = addIndexRefreshBlockToSomeIndices(indices);
93+
94+
int expectedHits = 0;
95+
for (String index : unblockedIndices) {
96+
expectedHits += numDocsPerIndex.get(index);
97+
}
98+
99+
final long expectedHitsL = expectedHits;
100+
assertResponse(
101+
client().prepareMultiSearch()
102+
.add(prepareSearch().setQuery(QueryBuilders.matchAllQuery()))
103+
.add(prepareSearch().setQuery(QueryBuilders.termQuery("field", "blah"))),
104+
response -> {
105+
assertHitCount(Objects.requireNonNull(response.getResponses()[0].getResponse()), expectedHitsL);
106+
assertHitCount(Objects.requireNonNull(response.getResponses()[1].getResponse()), 0);
107+
}
108+
);
109+
}
110+
111+
public void testSearchShardsOnIndicesWithIndexRefreshBlocks() {
112+
List<String> indices = createIndices();
113+
indexDocuments(indices);
114+
List<String> unblockedIndices = addIndexRefreshBlockToSomeIndices(indices);
115+
116+
var resp = client().execute(
117+
TransportSearchShardsAction.TYPE,
118+
new SearchShardsRequest(
119+
indices.toArray(new String[0]),
120+
IndicesOptions.DEFAULT,
121+
new MatchAllQueryBuilder(),
122+
null,
123+
null,
124+
true,
125+
null
126+
)
127+
).actionGet();
128+
for (SearchShardsGroup group : resp.getGroups()) {
129+
assertTrue(unblockedIndices.contains(group.shardId().getIndex().getName()));
130+
}
131+
}
132+
133+
private List<String> createIndices() {
134+
int numIndices = randomIntBetween(1, 3);
135+
List<String> indices = new ArrayList<>();
136+
for (int i = 0; i < numIndices; i++) {
137+
indices.add("test" + i);
138+
createIndex("test" + i);
139+
}
140+
return indices;
141+
}
142+
143+
private Map<String, Integer> indexDocuments(List<String> indices) {
144+
Map<String, Integer> numDocsPerIndex = new HashMap<>();
145+
List<IndexRequestBuilder> indexRequests = new ArrayList<>();
146+
for (String index : indices) {
147+
int numDocs = randomIntBetween(0, 10);
148+
numDocsPerIndex.put(index, numDocs);
149+
for (int i = 0; i < numDocs; i++) {
150+
indexRequests.add(prepareIndex(index).setId(String.valueOf(i)).setSource("field", "value"));
151+
}
152+
}
153+
indexRandom(true, indexRequests);
154+
155+
return numDocsPerIndex;
156+
}
157+
158+
private List<String> addIndexRefreshBlockToSomeIndices(List<String> indices) {
159+
List<String> unblockedIndices = new ArrayList<>();
160+
var blocksBuilder = ClusterBlocks.builder().blocks(EMPTY_CLUSTER_BLOCK);
161+
for (String index : indices) {
162+
boolean blockIndex = randomBoolean();
163+
if (blockIndex) {
164+
blocksBuilder.addIndexBlock(ProjectId.DEFAULT, index, IndexMetadata.INDEX_REFRESH_BLOCK);
165+
} else {
166+
unblockedIndices.add(index);
167+
}
168+
}
169+
170+
var dataNodes = clusterService().state().getNodes().getAllNodes();
171+
for (DiscoveryNode dataNode : dataNodes) {
172+
ClusterService clusterService = internalCluster().getInstance(ClusterService.class, dataNode.getName());
173+
ClusterState currentState = clusterService.state();
174+
ClusterState newState = ClusterState.builder(currentState).blocks(blocksBuilder).build();
175+
setState(clusterService, newState);
176+
}
177+
178+
return unblockedIndices;
179+
}
180+
}

‎server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1950,6 +1950,7 @@ List<SearchShardIterator> getLocalShardsIterator(
19501950
Set<ResolvedExpression> indicesAndAliases,
19511951
String[] concreteIndices
19521952
) {
1953+
concreteIndices = ignoreBlockedIndices(projectState, concreteIndices);
19531954
var routingMap = indexNameExpressionResolver.resolveSearchRouting(
19541955
projectState.metadata(),
19551956
searchRequest.routing(),
@@ -1982,6 +1983,20 @@ List<SearchShardIterator> getLocalShardsIterator(
19821983
return Arrays.asList(list);
19831984
}
19841985

1986+
static String[] ignoreBlockedIndices(ProjectState projectState, String[] concreteIndices) {
1987+
// optimization: mostly we do not have any blocks so there's no point in the expensive per-index checking
1988+
boolean hasIndexBlocks = projectState.blocks().indices(projectState.projectId()).isEmpty() == false;
1989+
if (hasIndexBlocks) {
1990+
return Arrays.stream(concreteIndices)
1991+
.filter(
1992+
index -> projectState.blocks()
1993+
.hasIndexBlock(projectState.projectId(), index, IndexMetadata.INDEX_REFRESH_BLOCK) == false
1994+
)
1995+
.toArray(String[]::new);
1996+
}
1997+
return concreteIndices;
1998+
}
1999+
19852000
private interface TelemetryListener {
19862001
void setRemotes(int count);
19872002

0 commit comments

Comments
 (0)