Populate EsqlExecutionInfo.clusterInfo from field caps response

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java

@@ -10,8 +10,12 @@
 package org.elasticsearch.search.fieldcaps;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ResolvedIndexExpression;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IllegalIndexShardStateException;
@@ -21,18 +25,21 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
 
@@ -266,4 +273,144 @@ public void testReturnAllLocal() {
             }
         }
     }
+
+    public void testResolvedToMatchingEverywhere() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
+            .setFields("*")
+            .setIncludeResolvedTo(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster));
+
+        ResolvedIndexExpressions local = response.getResolvedLocally();
+        assertThat(local, notNullValue());
+        assertThat(local.expressions(), hasSize(1));
+        assertEquals(
+            local.expressions().get(0).localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
+        );
+
+        List<String> localIndicesList = local.getLocalIndicesList();
+        assertThat(localIndicesList, hasSize(1));
+        assertThat(localIndicesList, containsInAnyOrder(localIndex));
+
+        Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
+        assertThat(remote, notNullValue());
+        assertThat(remote, aMapWithSize(1));
+        assertThat(remote.keySet(), contains(remoteClusterAlias));
+
+        ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
+        List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
+        assertThat(remoteIndicesList, hasSize(1));
+        assertEquals(
+            remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
+        );
+        assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
+    }
+
+    public void testResolvedToMatchingLocallyOnly() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        String nonExistentIndex = "non-existent-index";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster)
+            .setFields("*")
+            .setIncludeResolvedTo(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex));
+
+        ResolvedIndexExpressions local = response.getResolvedLocally();
+        assertThat(local, notNullValue());
+        assertThat(local.expressions(), hasSize(1));
+        assertEquals(
+            local.expressions().get(0).localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
+        );
+
+        List<String> localIndicesList = local.getLocalIndicesList();
+        assertThat(localIndicesList, hasSize(1));
+        assertThat(localIndicesList, containsInAnyOrder(localIndex));
+
+        Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
+        assertThat(remote, notNullValue());
+        assertThat(remote, aMapWithSize(1));
+        assertThat(remote.keySet(), contains(remoteClusterAlias));
+
+        ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
+        List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
+        assertThat(remoteIndicesList, hasSize(0));
+        List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
+        assertEquals(1, remoteResolvedExpressions.size());
+        assertEquals(
+            remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
+        );
+        assertEquals(0, remoteIndicesList.size());
+    }
+
+    public void testResolvedToMatchingRemotelyOnly() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        String nonExistentIndex = "non-existent-index";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex);
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster)
+            .setFields("*")
+            .setIncludeResolvedTo(true)
+            .setIndicesOptions(options) // without ignore unavaliable would throw error
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster));
+
+        ResolvedIndexExpressions local = response.getResolvedLocally();
+        assertThat(local, notNullValue());
+        assertThat(local.expressions(), hasSize(1));
+        assertEquals(
+            local.expressions().get(0).localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE
+        );
+
+        List<String> localIndicesList = local.getLocalIndicesList();
+        assertThat(localIndicesList, hasSize(0));
+
+        Map<String, ResolvedIndexExpressions> remote = response.getResolvedRemotely();
+        assertThat(remote, notNullValue());
+        assertThat(remote, aMapWithSize(1));
+        assertThat(remote.keySet(), contains(remoteClusterAlias));
+
+        ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias);
+        List<String> remoteIndicesList = remoteResponse.getLocalIndicesList();
+        assertThat(remoteIndicesList, hasSize(1));
+        assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex));
+        List<ResolvedIndexExpression> remoteResolvedExpressions = remoteResponse.expressions();
+        assertEquals(1, remoteResolvedExpressions.size());
+        ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0);
+        assertEquals(
+            remoteExpression.localExpressions().localIndexResolutionResult(),
+            ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS
+        );
+        assertEquals(1, remoteExpression.localExpressions().indices().size());
+        assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original());
+    }
+
+    public void testIncludesMinTransportVersion() {
+        if (randomBoolean()) {
+            assertAcked(client().admin().indices().prepareCreate("index"));
+        }
+        var response = client().prepareFieldCaps("_all").setFields("*").get();
+        assertThat(response.minTransportVersion(), equalTo(TransportVersion.current()));
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -13,6 +13,8 @@
 import org.apache.http.entity.StringEntity;
 import org.apache.logging.log4j.Level;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ResolvedIndexExpression;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -102,6 +104,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -913,6 +916,154 @@ public void testIndexMode() throws Exception {
         assertThat(actualIndexModes, equalTo(indexModes));
     }
 
+    public void testResolvedExpressionWithIndexAlias() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get();
+        assertIndices(response, "new_index");
+
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(1, resolvedLocally.expressions().size());
+        ResolvedIndexExpression expression = expressions.get(0);
+        assertEquals("current", expression.original());
+        Set<String> concreteIndices = expression.localExpressions().indices();
+        assertEquals(1, concreteIndices.size());
+        assertTrue(concreteIndices.contains("new_index"));
+    }
+
+    public void testResolvedExpressionWithWildcard() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get();
+        assertIndices(response, "new_index", "old_index");
+
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(1, resolvedLocally.expressions().size());
+        ResolvedIndexExpression expression = expressions.get(0);
+        assertEquals("*index", expression.original());
+        Set<String> concreteIndices = expression.localExpressions().indices();
+        assertEquals(2, concreteIndices.size());
+        assertTrue(concreteIndices.containsAll(Set.of("new_index", "old_index")));
+    }
+
+    public void testResolvedExpressionWithClosedIndices() throws IOException {
+        // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
+        assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error"));
+        ensureGreen("index1-error", "index2-error");
+
+        // Closed shards will result to index error because shards must be in readable state
+        closeShards(internalCluster(), "index1-error", "index2-error");
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error")
+            .setFields("*")
+            .setIncludeResolvedTo(true)
+            .get();
+        Set<String> openIndices = Set.of("old_index", "new_index");
+        Set<String> closedIndices = Set.of("index1-error", "index2-error");
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(4, resolvedLocally.expressions().size());
+        for (ResolvedIndexExpression expression : expressions) {
+            ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
+            if (openIndices.contains(expression.original())) {
+                Set<String> concreteIndices = localExpressions.indices();
+                assertEquals(1, concreteIndices.size());
+                assertTrue(concreteIndices.contains(expression.original())); // no aliases here, so the concrete index == original index
+                assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
+            } else if (closedIndices.contains(expression.original())) {
+                Set<String> concreteIndices = localExpressions.indices();
+                assertEquals(0, concreteIndices.size());
+                assertEquals(
+                    ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
+                    localExpressions.localIndexResolutionResult()
+                );
+            }
+        }
+    }
+
+    public void testResolvedExpressionWithAllIndices() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get();
+        assertIndices(response, "new_index", "old_index");
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(1, resolvedLocally.expressions().size());
+        ResolvedIndexExpression expression = expressions.get(0);
+        assertEquals("_all", expression.original()); // not setting indices means _all
+        ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
+        Set<String> concreteIndices = localExpressions.indices();
+        assertTrue(concreteIndices.containsAll(Set.of("new_index", "old_index")));
+        assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
+    }
+
+    public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index")
+            .setFields("*")
+            .setIndicesOptions(options)
+            .setIncludeResolvedTo(true)
+            .get();
+
+        assertIndices(response);
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(1, expressions.size());
+        ResolvedIndexExpression expression = expressions.get(0);
+        assertEquals("old_index", expression.original());
+        assertEquals(1, resolvedLocally.expressions().size());
+        ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
+        Set<String> concreteIndices = localExpressions.indices();
+        assertEquals(0, concreteIndices.size());
+        assertEquals(
+            ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE,
+            localExpressions.localIndexResolutionResult()
+        );
+    }
+
+    public void testResolvedExpressionWithIndexFilter() throws InterruptedException {
+        assertAcked(
+            prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"),
+            prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")
+        );
+
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+        reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08"));
+        reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08"));
+        reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12"));
+        reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08"));
+        indexRandom(true, reqs);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*")
+            .setFields("*")
+            .setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01"))
+            .setIncludeResolvedTo(true)
+            .get();
+
+        assertIndices(response, "index-2");
+        assertEquals(0, response.getResolvedRemotely().size());
+        ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally();
+        List<ResolvedIndexExpression> expressions = resolvedLocally.expressions();
+        assertEquals(1, resolvedLocally.expressions().size());
+        ResolvedIndexExpression expression = expressions.get(0);
+        assertEquals("index-*", expression.original());
+        ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions();
+        Set<String> concreteIndices = localExpressions.indices();
+        assertEquals(2, concreteIndices.size());
+        assertTrue(concreteIndices.containsAll(Set.of("index-1", "index-2")));
+        assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult());
+    }
+
+    public void testNoneExpressionIndices() {
+        // The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get();
+
+        assertThat(response.getIndices().length, is(0));
+    }
+
     private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
         assertNotNull(response.getIndices());
         Arrays.sort(indices);