Skip to content

Commit 07a9bd2

Browse files
phananh1010phananh1010
committed
Mirror upstream elastic#136632 as single snapshot commit for AI review
BASE=ec0efafc2fdc49adffe4f58806ea71d28b7d9f49 HEAD=171872a6ca36f8594f0e7fa7ad2981aa23a8d1a2 Branch=main
1 parent ec0efaf commit 07a9bd2

File tree

23 files changed

+720
-90
lines changed

23 files changed

+720
-90
lines changed

‎docs/changelog/136632.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 136632
2+
summary: Field caps transport changes to return for each original expression what
3+
it was resolved to
4+
area: Search
5+
type: enhancement
6+
issues: []

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,11 @@
1111

1212
import org.elasticsearch.ExceptionsHelper;
1313
import org.elasticsearch.TransportVersion;
14+
import org.elasticsearch.action.ResolvedIndexExpression;
15+
import org.elasticsearch.action.ResolvedIndexExpressions;
1416
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
1517
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
18+
import org.elasticsearch.action.support.IndicesOptions;
1619
import org.elasticsearch.client.internal.Client;
1720
import org.elasticsearch.common.settings.Settings;
1821
import org.elasticsearch.index.shard.IllegalIndexShardStateException;
@@ -22,18 +25,21 @@
2225
import java.io.IOException;
2326
import java.util.Arrays;
2427
import java.util.List;
28+
import java.util.Map;
2529

2630
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2731
import static org.hamcrest.Matchers.aMapWithSize;
2832
import static org.hamcrest.Matchers.arrayContaining;
2933
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
34+
import static org.hamcrest.Matchers.contains;
3035
import static org.hamcrest.Matchers.containsInAnyOrder;
3136
import static org.hamcrest.Matchers.emptyArray;
3237
import static org.hamcrest.Matchers.equalTo;
3338
import static org.hamcrest.Matchers.hasItems;
3439
import static org.hamcrest.Matchers.hasKey;
3540
import static org.hamcrest.Matchers.hasSize;
3641
import static org.hamcrest.Matchers.not;
42+
import static org.hamcrest.Matchers.notNullValue;
3743

3844
public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
3945

@@ -268,6 +274,138 @@ public void testReturnAllLocal() {
268274
}
269275
}
270276

277+
public void testResolvedToMatchingEverywhere() {
278+
String localIndex = "index-local";
279+
String remoteIndex = "index-remote";
280+
String remoteClusterAlias = "remote_cluster";
281+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
282+
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
283+
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
284+
.setFields("*")
285+
.setIncludeResolvedTo(true)
286+
.get();
287+
288+
assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster));
289+
290+
ResolvedIndexExpressions local = response.getResolvedLocally();
291+
assertThat(local, notNullValue());
292+
assertThat(local.expressions(), hasSize(1));
293+
assertEquals(
294+
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
295+
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
296+
);
297+
298+
List<String> localIndicesList = local.getLocalIndicesList();
299+
assertThat(localIndicesList, hasSize(1));
300+
assertThat(localIndicesList, containsInAnyOrder(localIndex));
301+
302+
Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
303+
assertThat(remote, notNullValue());
304+
assertThat(remote, aMapWithSize(1));
305+
assertThat(remote.keySet(), contains(remoteClusterAlias));
306+
307+
ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
308+
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
309+
assertThat(remoteIndicesList, hasSize(1));
310+
assertEquals(
311+
remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(),
312+
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
313+
);
314+
assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
315+
}
316+
317+
public void testResolvedToMatchingLocallyOnly() {
318+
String localIndex = "index-local";
319+
String remoteIndex = "index-remote";
320+
String remoteClusterAlias = "remote_cluster";
321+
String nonExistentIndex = "non-existent-index";
322+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
323+
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex);
324+
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
325+
.setFields("*")
326+
.setIncludeResolvedTo(true)
327+
.get();
328+
329+
assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex));
330+
331+
ResolvedIndexExpressions local = response.getResolvedLocally();
332+
assertThat(local, notNullValue());
333+
assertThat(local.expressions(), hasSize(1));
334+
assertEquals(
335+
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
336+
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
337+
);
338+
339+
List<String> localIndicesList = local.getLocalIndicesList();
340+
assertThat(localIndicesList, hasSize(1));
341+
assertThat(localIndicesList, containsInAnyOrder(localIndex));
342+
343+
Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
344+
assertThat(remote, notNullValue());
345+
assertThat(remote, aMapWithSize(1));
346+
assertThat(remote.keySet(), contains(remoteClusterAlias));
347+
348+
ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
349+
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
350+
assertThat(remoteIndicesList, hasSize(0));
351+
List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
352+
assertEquals(1, remoteResolvedExpressions.size());
353+
assertEquals(
354+
remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(),
355+
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
356+
);
357+
assertEquals(0, remoteIndicesList.size());
358+
}
359+
360+
public void testResolvedToMatchingRemotelyOnly() {
361+
String localIndex = "index-local";
362+
String remoteIndex = "index-remote";
363+
String remoteClusterAlias = "remote_cluster";
364+
String nonExistentIndex = "non-existent-index";
365+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
366+
String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
367+
boolean ignoreUnavailable = true;
368+
IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
369+
370+
FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster)
371+
.setFields("*")
372+
.setIncludeResolvedTo(true)
373+
.setIndicesOptions(options) // without ignore unavaliable would throw error
374+
.get();
375+
376+
assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster));
377+
378+
ResolvedIndexExpressions local = response.getResolvedLocally();
379+
assertThat(local, notNullValue());
380+
assertThat(local.expressions(), hasSize(1));
381+
assertEquals(
382+
local.expressions().get(0).localExpressions().localIndexResolutionResult(),
383+
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
384+
);
385+
386+
List<String> localIndicesList = local.getLocalIndicesList();
387+
assertThat(localIndicesList, hasSize(0));
388+
389+
Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
390+
assertThat(remote, notNullValue());
391+
assertThat(remote, aMapWithSize(1));
392+
assertThat(remote.keySet(), contains(remoteClusterAlias));
393+
394+
ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
395+
List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
396+
assertThat(remoteIndicesList, hasSize(1));
397+
assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
398+
List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
399+
assertEquals(1, remoteResolvedExpressions.size());
400+
ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0);
401+
assertEquals(
402+
remoteExpression.localExpressions().localIndexResolutionResult(),
403+
ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
404+
);
405+
assertEquals(1, remoteExpression.localExpressions().indices().size());
406+
assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original());
407+
}
408+
271409
public void testIncludesMinTransportVersion() {
272410
if (randomBoolean()) {
273411
assertAcked(client().admin().indices().prepareCreate("index"));

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import org.apache.http.entity.StringEntity;
1414
import org.apache.logging.log4j.Level;
1515
import org.elasticsearch.ElasticsearchException;
16+
import org.elasticsearch.action.ResolvedIndexExpression;
17+
import org.elasticsearch.action.ResolvedIndexExpressions;
1618
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
1719
import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
1820
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -102,6 +104,7 @@
102104
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
103105
import static org.hamcrest.Matchers.hasKey;
104106
import static org.hamcrest.Matchers.hasSize;
107+
import static org.hamcrest.Matchers.is;
105108
import static org.hamcrest.Matchers.not;
106109
import static org.hamcrest.Matchers.nullValue;
107110

@@ -913,6 +916,170 @@ public void testIndexMode() throws Exception {
913916
assertThat(actualIndexModes, equalTo(indexModes));
914917
}
915918

919+
public void testResolvedExpressionWithIndexAlias() {
920+
FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get();
921+
assertIndices(response, "new_index");
922+
923+
assertEquals(0, response.getResolvedRemotely().size());
924+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
925+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
926+
assertEquals(1, resolvedLocally.expressions().size());
927+
ResolvedIndexExpression expression = expressions.get(0);
928+
assertEquals("current", expression.original());
929+
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index"));
930+
}
931+
932+
public void testResolvedExpressionWithWildcard() {
933+
FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get();
934+
assertIndices(response, "new_index", "old_index");
935+
936+
assertEquals(0, response.getResolvedRemotely().size());
937+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
938+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
939+
assertEquals(1, resolvedLocally.expressions().size());
940+
ResolvedIndexExpression expression = expressions.get(0);
941+
assertEquals("*index", expression.original());
942+
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index"));
943+
}
944+
945+
public void testResolvedExpressionWithClosedIndices() throws IOException {
946+
// in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
947+
assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error"));
948+
ensureGreen("index1-error", "index2-error");
949+
950+
// Closed shards will result to index error because shards must be in readable state
951+
closeShards(internalCluster(), "index1-error", "index2-error");
952+
953+
FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error")
954+
.setFields("*")
955+
.setIncludeResolvedTo(true)
956+
.get();
957+
Set<String> openIndices = Set.of("old_index", "new_index");
958+
Set<String> closedIndices = Set.of("index1-error", "index2-error");
959+
assertEquals(0, response.getResolvedRemotely().size());
960+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
961+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
962+
assertEquals(4, resolvedLocally.expressions().size());
963+
for (ResolvedIndexExpression expression : expressions) {
964+
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
965+
if (openIndices.contains(expression.original())) {
966+
assertThat(expression.localExpressions().indices(), containsInAnyOrder(expression.original()));
967+
assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
968+
} else if (closedIndices.contains(expression.original())) {
969+
Set<String> concreteIndices = localExpressions.indices();
970+
assertEquals(0, concreteIndices.size());
971+
assertEquals(
972+
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
973+
localExpressions.localIndexResolutionResult()
974+
);
975+
}
976+
}
977+
}
978+
979+
public void testResolvedExpressionWithAllIndices() {
980+
FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get();
981+
assertIndices(response, "new_index", "old_index");
982+
assertEquals(0, response.getResolvedRemotely().size());
983+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
984+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
985+
assertEquals(1, resolvedLocally.expressions().size());
986+
ResolvedIndexExpression expression = expressions.get(0);
987+
assertEquals("_all", expression.original()); // not setting indices means _all
988+
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
989+
assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index"));
990+
assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
991+
}
992+
993+
public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() {
994+
boolean ignoreUnavailable = true;
995+
IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
996+
client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
997+
FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index")
998+
.setFields("*")
999+
.setIndicesOptions(options)
1000+
.setIncludeResolvedTo(true)
1001+
.get();
1002+
1003+
assertIndices(response);
1004+
assertEquals(0, response.getResolvedRemotely().size());
1005+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
1006+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
1007+
assertEquals(1, expressions.size());
1008+
ResolvedIndexExpression expression = expressions.get(0);
1009+
assertEquals("old_index", expression.original());
1010+
assertEquals(1, resolvedLocally.expressions().size());
1011+
ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
1012+
Set<String> concreteIndices = localExpressions.indices();
1013+
assertEquals(0, concreteIndices.size());
1014+
assertEquals(
1015+
ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
1016+
localExpressions.localIndexResolutionResult()
1017+
);
1018+
}
1019+
1020+
public void testResolvedExpressionWithIndexFilter() throws InterruptedException {
1021+
assertAcked(
1022+
prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"),
1023+
prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")
1024+
);
1025+
1026+
List<IndexRequestBuilder> reqs = new ArrayList<>();
1027+
reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08"));
1028+
reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08"));
1029+
reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12"));
1030+
reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08"));
1031+
indexRandom(true, reqs);
1032+
1033+
FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*")
1034+
.setFields("*")
1035+
.setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01"))
1036+
.setIncludeResolvedTo(true)
1037+
.get();
1038+
1039+
assertIndices(response, "index-2");
1040+
assertEquals(0, response.getResolvedRemotely().size());
1041+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
1042+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
1043+
assertEquals(1, resolvedLocally.expressions().size());
1044+
ResolvedIndexExpression expression = expressions.get(0);
1045+
assertEquals("index-*", expression.original());
1046+
assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-1", "index-2"));
1047+
}
1048+
1049+
public void testNoneExpressionIndices() {
1050+
// The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices
1051+
FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get();
1052+
1053+
assertThat(response.getIndices().length, is(0));
1054+
}
1055+
1056+
public void testExclusion() {
1057+
assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025"));
1058+
1059+
prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get();
1060+
prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get();
1061+
1062+
var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").get();
1063+
assertIndices(response, "index-2024");
1064+
}
1065+
1066+
public void testExclusionWithResolvedTo() {
1067+
assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025"));
1068+
1069+
prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get();
1070+
prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get();
1071+
1072+
var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").setIncludeResolvedTo(true).get();
1073+
assertIndices(response, "index-2024");
1074+
assertEquals(0, response.getResolvedRemotely().size());
1075+
ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
1076+
List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
1077+
assertEquals(1, resolvedLocally.expressions().size());
1078+
ResolvedIndexExpression expression = expressions.get(0);
1079+
assertEquals("index-*", expression.original());
1080+
assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-2024", "index-2025"));
1081+
}
1082+
9161083
private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
9171084
assertNotNull(response.getIndices());
9181085
Arrays.sort(indices);

0 commit comments

Comments
 (0)