diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 541fe0f869f26..25fe5bdb222cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -10,8 +10,12 @@ package org.elasticsearch.search.fieldcaps; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -21,11 +25,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; @@ -33,6 +39,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { @@ -266,4 +273,144 @@ public void testReturnAllLocal() { } } } + + public void testResolvedToMatchingEverywhere() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertEquals( + remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + } + + public void testResolvedToMatchingLocallyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(0)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + assertEquals( + remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + assertEquals(0, remoteIndicesList.size()); + } + + public void testResolvedToMatchingRemotelyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + + FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .setIndicesOptions(options) // without ignore unavaliable would throw error + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(0)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0); + assertEquals( + remoteExpression.localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertEquals(1, remoteExpression.localExpressions().indices().size()); + assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original()); + } + + public void testIncludesMinTransportVersion() { + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareCreate("index")); + } + var response = client().prepareFieldCaps("_all").setFields("*").get(); + assertThat(response.minTransportVersion(), equalTo(TransportVersion.current())); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index b9fcec556213a..9ff35b3d1331a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -13,6 +13,8 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -102,6 +104,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -913,6 +916,154 @@ public void testIndexMode() throws Exception { assertThat(actualIndexModes, equalTo(indexModes)); } + public void testResolvedExpressionWithIndexAlias() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("current", expression.original()); + Set concreteIndices = expression.localExpressions().indices(); + assertEquals(1, concreteIndices.size()); + assertTrue(concreteIndices.contains("new_index")); + } + + public void testResolvedExpressionWithWildcard() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("*index", expression.original()); + Set concreteIndices = expression.localExpressions().indices(); + assertEquals(2, concreteIndices.size()); + assertTrue(concreteIndices.containsAll(Set.of("new_index", "old_index"))); + } + + public void testResolvedExpressionWithClosedIndices() throws IOException { + // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite + assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); + ensureGreen("index1-error", "index2-error"); + + // Closed shards will result to index error because shards must be in readable state + closeShards(internalCluster(), "index1-error", "index2-error"); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error") + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + Set openIndices = Set.of("old_index", "new_index"); + Set closedIndices = Set.of("index1-error", "index2-error"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(4, resolvedLocally.expressions().size()); + for (ResolvedIndexExpression expression : expressions) { + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + if (openIndices.contains(expression.original())) { + Set concreteIndices = localExpressions.indices(); + assertEquals(1, concreteIndices.size()); + assertTrue(concreteIndices.contains(expression.original())); // no aliases here, so the concrete index == original index + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } else if (closedIndices.contains(expression.original())) { + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + } + } + + public void testResolvedExpressionWithAllIndices() { + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("_all", expression.original()); // not setting indices means _all + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertTrue(concreteIndices.containsAll(Set.of("new_index", "old_index"))); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } + + public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index") + .setFields("*") + .setIndicesOptions(options) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, expressions.size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("old_index", expression.original()); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + + public void testResolvedExpressionWithIndexFilter() throws InterruptedException { + assertAcked( + prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"), + prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long") + ); + + List reqs = new ArrayList<>(); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08")); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08")); + indexRandom(true, reqs); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*") + .setFields("*") + .setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01")) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response, "index-2"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertEquals(2, concreteIndices.size()); + assertTrue(concreteIndices.containsAll(Set.of("index-1", "index-2"))); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } + + public void testNoneExpressionIndices() { + // The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices + FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get(); + + assertThat(response.getIndices().length, is(0)); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java index 301db629d8d69..f7daaa0ff1255 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -41,7 +42,11 @@ public List getRemoteIndicesList() { } public static Builder builder() { - return new Builder(); + return new Builder(new ArrayList<>()); + } + + public static Builder threadSafeBuilder() { + return new Builder(Collections.synchronizedList(new ArrayList<>())); } @Override @@ -50,7 +55,11 @@ public void writeTo(StreamOutput out) throws IOException { } public static final class Builder { - private final List expressions = new ArrayList<>(); + private final List expressions; + + public Builder(List expressions) { + this.expressions = expressions; + } /** * Add a new resolved expression. @@ -73,6 +82,14 @@ public void addExpressions( ); } + /** + * Add a new resolved expression. + * @param expression the expression you want to add. + */ + public void addExpression(ResolvedIndexExpression expression) { + expressions.add(expression); + } + public void addRemoteExpressions(String original, Set remoteExpressions) { Objects.requireNonNull(original); Objects.requireNonNull(remoteExpressions); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 4ed4e24110cd9..c1d94cb774e33 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -40,6 +40,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = TransportVersion.fromName("field_caps_add_cluster_alias"); + private static final TransportVersion RESOLVED_FIELDS_CAPS = TransportVersion.fromName("resolved_fields_caps"); private String clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; @@ -58,6 +59,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen */ private transient boolean includeIndices = false; + private boolean includeResolvedTo = false; + /** * Controls whether all local indices should be returned if no remotes matched * See {@link org.elasticsearch.transport.RemoteClusterService#groupIndices} returnLocalAll argument. @@ -93,6 +96,11 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException { } else { clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; } + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + includeResolvedTo = in.readBoolean(); + } else { + includeResolvedTo = false; + } } public FieldCapabilitiesRequest() {} @@ -145,6 +153,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(FIELD_CAPS_ADD_CLUSTER_ALIAS)) { out.writeOptionalString(clusterAlias); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeBoolean(includeResolvedTo); + } } @Override @@ -223,6 +234,11 @@ public FieldCapabilitiesRequest includeIndices(boolean includeIndices) { return this; } + public FieldCapabilitiesRequest includeResolvedTo(boolean includeResolvedTo) { + this.includeResolvedTo = includeResolvedTo; + return this; + } + public FieldCapabilitiesRequest returnLocalAll(boolean returnLocalAll) { this.returnLocalAll = returnLocalAll; return this; @@ -256,6 +272,10 @@ public boolean includeIndices() { return includeIndices; } + public boolean includeResolvedTo() { + return includeResolvedTo; + } + public boolean returnLocalAll() { return returnLocalAll; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 4437895c7e08d..9e79c52ce8ae1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -63,4 +63,9 @@ public FieldCapabilitiesRequestBuilder setReturnLocalAll(boolean returnLocalAll) request().returnLocalAll(returnLocalAll); return this; } + + public FieldCapabilitiesRequestBuilder setIncludeResolvedTo(boolean resolvedTo) { + request().includeResolvedTo(resolvedTo); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index e3fcd5bbd200d..b7f5d225ab19e 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -9,63 +9,81 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * Response for {@link FieldCapabilitiesRequest} requests. */ public class FieldCapabilitiesResponse extends ActionResponse implements ChunkedToXContentObject { + public static final ParseField INDICES_FIELD = new ParseField("indices"); public static final ParseField FIELDS_FIELD = new ParseField("fields"); private static final ParseField FAILED_INDICES_FIELD = new ParseField("failed_indices"); public static final ParseField FAILURES_FIELD = new ParseField("failures"); + private static final TransportVersion RESOLVED_FIELDS_CAPS = TransportVersion.fromName("resolved_fields_caps"); + private final String[] indices; + private final ResolvedIndexExpressions resolvedLocally; + private final transient Map resolvedRemotely; private final Map> fields; private final List failures; private final List indexResponses; + private final TransportVersion minTransportVersion; - public FieldCapabilitiesResponse( - String[] indices, - Map> fields, - List failures - ) { - this(indices, fields, Collections.emptyList(), failures); - } - - public FieldCapabilitiesResponse(String[] indices, Map> fields) { - this(indices, fields, Collections.emptyList(), Collections.emptyList()); + public static FieldCapabilitiesResponse empty() { + return new FieldCapabilitiesResponse( + Strings.EMPTY_ARRAY, + null, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyList(), + null + ); } - public FieldCapabilitiesResponse(List indexResponses, List failures) { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses, failures); + public static FieldCapabilitiesResponse.Builder builder() { + return new FieldCapabilitiesResponse.Builder(); } private FieldCapabilitiesResponse( String[] indices, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, Map> fields, List indexResponses, - List failures + List failures, + TransportVersion minTransportVersion ) { this.fields = Objects.requireNonNull(fields); + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = Objects.requireNonNull(resolvedRemotely); this.indexResponses = Objects.requireNonNull(indexResponses); this.indices = indices; this.failures = failures; + this.minTransportVersion = minTransportVersion; } public FieldCapabilitiesResponse(StreamInput in) throws IOException { @@ -73,6 +91,13 @@ public FieldCapabilitiesResponse(StreamInput in) throws IOException { this.fields = in.readMap(FieldCapabilitiesResponse::readField); this.indexResponses = FieldCapabilitiesIndexResponse.readList(in); this.failures = in.readCollectionAsList(FieldCapabilitiesFailure::new); + this.resolvedLocally = in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS) + ? in.readOptionalWriteable(ResolvedIndexExpressions::new) + : null; + this.resolvedRemotely = Collections.emptyMap(); + this.minTransportVersion = in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS) + ? in.readOptional(TransportVersion::readVersion) + : null; } /** @@ -115,6 +140,20 @@ public List getIndexResponses() { return indexResponses; } + /** + * Locally resolved index expressions + */ + public ResolvedIndexExpressions getResolvedLocally() { + return resolvedLocally; + } + + /** + * Locally resolved index expressions + */ + public Map getResolvedRemotely() { + return resolvedRemotely; + } + /** * * Get the field capabilities per type for the provided {@code field}. @@ -123,6 +162,14 @@ public Map getField(String field) { return fields.get(field); } + /** + * @return the minTransportVersion across all clusters involved in resolution + */ + @Nullable + public TransportVersion minTransportVersion() { + return minTransportVersion; + } + /** * Returns true if the provided field is a metadata field. */ @@ -144,6 +191,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(fields, FieldCapabilitiesResponse::writeField); FieldCapabilitiesIndexResponse.writeList(out, indexResponses); out.writeCollection(failures); + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeOptionalWriteable(resolvedLocally); + } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeOptional((Writer) (o, v) -> TransportVersion.writeVersion(v, o), minTransportVersion); + } } private static void writeField(StreamOutput out, Map map) throws IOException { @@ -182,23 +235,89 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; return Arrays.equals(indices, that.indices) + && Objects.equals(resolvedLocally, that.resolvedLocally) + && Objects.equals(resolvedRemotely, that.resolvedRemotely) && Objects.equals(fields, that.fields) && Objects.equals(indexResponses, that.indexResponses) - && Objects.equals(failures, that.failures); + && Objects.equals(failures, that.failures) + && Objects.equals(minTransportVersion, that.minTransportVersion); } @Override public int hashCode() { - int result = Objects.hash(fields, indexResponses, failures); - result = 31 * result + Arrays.hashCode(indices); - return result; + return Objects.hash(resolvedLocally, resolvedRemotely, fields, indexResponses, failures, minTransportVersion) * 31 + Arrays + .hashCode(indices); } @Override public String toString() { - if (indexResponses.size() > 0) { - return "FieldCapabilitiesResponse{unmerged}"; + return indexResponses.isEmpty() ? Strings.toString(this) : "FieldCapabilitiesResponse{unmerged}"; + } + + public static class Builder { + private String[] indices = Strings.EMPTY_ARRAY; + private ResolvedIndexExpressions resolvedLocally; + private Map resolvedRemotely = Collections.emptyMap(); + private Map> fields = Collections.emptyMap(); + private List indexResponses = Collections.emptyList(); + private List failures = Collections.emptyList(); + private TransportVersion minTransportVersion; + + private Builder() {} + + public Builder withIndices(String[] indices) { + this.indices = indices; + return this; + } + + public Builder withResolved(ResolvedIndexExpressions resolvedLocally, Map resolvedRemotely) { + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = resolvedRemotely; + return this; + } + + public Builder withResolvedRemotelyBuilder(Map resolvedRemotelyBuilder) { + this.resolvedRemotely = resolvedRemotelyBuilder.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); + return this; + } + + public Builder withResolvedLocally(ResolvedIndexExpressions resolvedLocally) { + this.resolvedLocally = resolvedLocally; + return this; + } + + public Builder withFields(Map> fields) { + this.fields = fields; + return this; + } + + public Builder withIndexResponses(Collection indexResponses) { + this.indexResponses = new ArrayList<>(indexResponses); + return this; + } + + public Builder withFailures(List failures) { + this.failures = failures; + return this; + } + + public Builder withMinTransportVersion(TransportVersion minTransportVersion) { + this.minTransportVersion = minTransportVersion; + return this; + } + + public FieldCapabilitiesResponse build() { + return new FieldCapabilitiesResponse( + indices, + resolvedLocally, + resolvedRemotely, + fields, + indexResponses, + failures, + minTransportVersion + ); } - return Strings.toString(this); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 1228c2d616b09..c296d3da65d14 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -13,24 +13,29 @@ import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.support.AbstractThreadedActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.Maps; @@ -67,8 +72,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -144,25 +151,62 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action final Executor singleThreadedExecutor = buildSingleThreadedExecutor(searchCoordinationExecutor, LOGGER); assert task instanceof CancellableTask; final CancellableTask fieldCapTask = (CancellableTask) task; - // retrieve the initial timestamp in case the action is a cross cluster search + // retrieve the initial timestamp in case the action is a cross-cluster search long nowInMillis = request.nowInMillis() == null ? System.currentTimeMillis() : request.nowInMillis(); final ProjectState projectState = projectResolver.getProjectState(clusterService.state()); + final var minTransportVersion = new AtomicReference<>(clusterService.state().getMinTransportVersion()); final Map remoteClusterIndices = transportService.getRemoteClusterService() .groupIndices(request.indicesOptions(), request.indices(), request.returnLocalAll()); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - final String[] concreteIndices; - if (localIndices == null) { - // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - concreteIndices = Strings.EMPTY_ARRAY; + + final List resolvedLocallyList; + if (request.getResolvedIndexExpressions() != null) { + // in CPS the SAF would populate resolvedExpressions for the local project + // TODO MP Might need to expand local indices? + resolvedLocallyList = request.getResolvedIndexExpressions().expressions(); } else { - concreteIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices); + resolvedLocallyList = new ArrayList<>(); + } + // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices + if (localIndices != null && resolvedLocallyList.isEmpty()) { + ProjectMetadata projectMetadata = projectState.metadata(); + IndicesOptions indicesOptions = localIndices.indicesOptions(); + String[] localIndexNames = localIndices.indices(); + if (localIndexNames.length == 0) { + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(projectMetadata, indicesOptions); + resolvedLocallyList.add(createResolvedIndexExpression(Metadata.ALL, concreteIndexNames)); + } else if (false == IndexNameExpressionResolver.isNoneExpression(localIndexNames)) { + for (String localIndexName : localIndexNames) { + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames( + projectMetadata, + indicesOptions, + localIndices.includeDataStreams(), + localIndexName + ); + resolvedLocallyList.add(createResolvedIndexExpression(localIndexName, concreteIndexNames)); + } + } } + String[] concreteIndices = resolvedLocallyList.stream() + .map(r -> r.localExpressions().indices()) + .flatMap(Set::stream) + .distinct() + .toArray(String[]::new); if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) { - listener.onResponse(new FieldCapabilitiesResponse(new String[0], Collections.emptyMap())); + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder(); + if (request.includeResolvedTo()) { // TODO MP is this ok for CCS/CPS or should we also add remote resolution? + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(resolvedLocallyList)); + } + responseBuilder.withMinTransportVersion(minTransportVersion.get()); + listener.onResponse(responseBuilder.build()); return; } + if (false == request.includeResolvedTo()) { + resolvedLocallyList.clear(); + } + checkIndexBlocks(projectState, concreteIndices); final FailureCollector indexFailures = new FailureCollector(); final Map indexResponses = new HashMap<>(); @@ -174,6 +218,10 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action indexResponses.clear(); indexMappingHashToResponses.clear(); }; + Map resolvedRemotely = new ConcurrentHashMap<>(); + for (String clusterAlias : remoteClusterIndices.keySet()) { + resolvedRemotely.put(clusterAlias, ResolvedIndexExpressions.threadSafeBuilder()); + } final Consumer handleIndexResponse = resp -> { if (fieldCapTask.isCancelled()) { releaseResourcesOnCancel.run(); @@ -219,6 +267,7 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action return; } indexFailures.collect(index, error); + if (fieldCapTask.isCancelled()) { releaseResourcesOnCancel.run(); } @@ -230,12 +279,22 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action LOGGER.trace("clear index responses on cancellation submitted"); } }); + try (RefCountingRunnable refs = new RefCountingRunnable(() -> { finishedOrCancelled.set(true); if (fieldCapTask.notifyIfCancelled(listener)) { releaseResourcesOnCancel.run(); } else { - mergeIndexResponses(request, fieldCapTask, indexResponses, indexFailures, listener); + mergeIndexResponses( + request, + fieldCapTask, + indexResponses, + indexFailures, + resolvedLocallyList, + resolvedRemotely, + minTransportVersion, + listener + ); } })) { // local cluster @@ -263,6 +322,17 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action OriginalIndices originalIndices = remoteIndices.getValue(); FieldCapabilitiesRequest remoteRequest = prepareRemoteRequest(clusterAlias, request, originalIndices, nowInMillis); ActionListener remoteListener = ActionListener.wrap(response -> { + + if (request.includeResolvedTo()) { + ResolvedIndexExpressions resolvedOnRemoteProject = response.getResolvedLocally(); + for (ResolvedIndexExpression remoteResolvedExpression : resolvedOnRemoteProject.expressions()) { + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(remoteResolvedExpression); + return v; + }); + } + } + for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()); handleIndexResponse.accept( @@ -279,11 +349,43 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action Exception ex = failure.getException(); for (String index : failure.getIndices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); } } + minTransportVersion.accumulateAndGet(response.minTransportVersion(), (lhs, rhs) -> { + if (lhs == null || rhs == null) { + return null; + } + return TransportVersion.min(lhs, rhs); + }); }, ex -> { for (String index : originalIndices.indices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); } }); @@ -319,6 +421,20 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action } } + private static ResolvedIndexExpression createResolvedIndexExpression(String original, String[] concreteIndexNames) { + boolean isWildcard = Regex.isSimpleMatchPattern(original); + // if it is a wildcard we consider it successful even if it didn't resolve to any concrete index + ResolvedIndexExpression.LocalIndexResolutionResult resolutionResult = concreteIndexNames.length > 0 || isWildcard + ? ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + : ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE; + + return new ResolvedIndexExpression( + original, + new ResolvedIndexExpression.LocalExpressions(Set.of(concreteIndexNames), resolutionResult, null), + Collections.emptySet() + ); + } + public static Executor buildSingleThreadedExecutor(Executor searchCoordinationExecutor, Logger logger) { final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner("field_caps", 1, searchCoordinationExecutor); return r -> throttledTaskRunner.enqueueTask(new ActionListener<>() { @@ -360,35 +476,56 @@ private static void mergeIndexResponses( CancellableTask task, Map indexResponses, FailureCollector indexFailures, + List resolvedLocallyList, + Map resolvedRemotely, + AtomicReference minTransportVersion, ActionListener listener ) { + ResolvedIndexExpressions resolvedLocally = new ResolvedIndexExpressions(resolvedLocallyList); List failures = indexFailures.build(indexResponses.keySet()); - if (indexResponses.size() > 0) { + if (indexResponses.isEmpty() == false) { if (request.isMergeResults()) { - ActionListener.completeWith(listener, () -> merge(indexResponses, task, request, failures)); + ActionListener.completeWith( + listener, + () -> merge(request, task, indexResponses, resolvedLocally, resolvedRemotely, failures, minTransportVersion) + ); } else { - listener.onResponse(new FieldCapabilitiesResponse(new ArrayList<>(indexResponses.values()), failures)); + listener.onResponse( + FieldCapabilitiesResponse.builder() + .withIndexResponses(indexResponses.values()) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withFailures(failures) + .withMinTransportVersion(minTransportVersion.get()) + .build() + ); } - } else { - // we have no responses at all, maybe because of errors - if (indexFailures.isEmpty() == false) { - /* - * Under no circumstances are we to pass timeout errors originating from SubscribableListener as top-level errors. - * Instead, they should always be passed through the response object, as part of "failures". - */ - if (failures.stream() - .anyMatch( - failure -> failure.getException() instanceof IllegalStateException ise - && ise.getCause() instanceof ElasticsearchTimeoutException - )) { - listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyList(), failures)); - } else { - // throw back the first exception - listener.onFailure(failures.get(0).getException()); - } + } else if (indexFailures.isEmpty() == false) { + /* + * Under no circumstances are we to pass timeout errors originating from SubscribableListener as top-level errors. + * Instead, they should always be passed through the response object, as part of "failures". + */ + if (failures.stream() + .anyMatch( + failure -> failure.getException() instanceof IllegalStateException ise + && ise.getCause() instanceof ElasticsearchTimeoutException + )) { + listener.onResponse( + FieldCapabilitiesResponse.builder() + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withFailures(failures) + .withMinTransportVersion(minTransportVersion.get()) + .build() + + ); } else { - listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyList(), Collections.emptyList())); + // throw back the first exception + listener.onFailure(failures.get(0).getException()); } + } else { + // TODO MP is this correct or should we add locally resolved if setResolved is true? + listener.onResponse(FieldCapabilitiesResponse.empty()); } } @@ -410,6 +547,7 @@ public static FieldCapabilitiesRequest prepareRemoteRequest( remoteRequest.indexFilter(request.indexFilter()); remoteRequest.nowInMillis(nowInMillis); remoteRequest.includeEmptyFields(request.includeEmptyFields()); + remoteRequest.includeResolvedTo(request.includeResolvedTo()); return remoteRequest; } @@ -420,10 +558,13 @@ private static boolean hasSameMappingHash(FieldCapabilitiesIndexResponse r1, Fie } private static FieldCapabilitiesResponse merge( - Map indexResponsesMap, - CancellableTask task, FieldCapabilitiesRequest request, - List failures + CancellableTask task, + Map indexResponsesMap, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, + List failures, + AtomicReference minTransportVersion ) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker task.ensureNotCancelled(); @@ -453,6 +594,22 @@ private static FieldCapabilitiesResponse merge( collectFields(fieldsBuilder, fields, request.includeIndices()); } + List failedIndices = failures.stream().flatMap(f -> Arrays.stream(f.getIndices())).toList(); + List collect = resolvedLocally.expressions().stream().map(expression -> { + if (failedIndices.contains(expression.original())) { + return new ResolvedIndexExpression( + expression.original(), + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + // TODO MP maybe with can be auth too and maybe can remain a success if wildcard? + null + ), + expression.remoteExpressions() + ); + } + return expression; + }).collect(Collectors.toList()); // The merge method is only called on the primary coordinator for cross-cluster field caps, so we // log relevant "5xx" errors that occurred in this 2xx response to ensure they are only logged once. // These failures have already been deduplicated, before this method was called. @@ -464,7 +621,15 @@ private static FieldCapabilitiesResponse merge( ); } } - return new FieldCapabilitiesResponse(indices, Collections.unmodifiableMap(fields), failures); + + return FieldCapabilitiesResponse.builder() + .withIndices(indices) + .withResolvedLocally(new ResolvedIndexExpressions(collect)) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withFields(fields) + .withFailures(failures) + .withMinTransportVersion(minTransportVersion.get()) + .build(); } private static boolean shouldLogException(Exception e) { diff --git a/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv new file mode 100644 index 0000000000000..773be2c2c150c --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv @@ -0,0 +1 @@ +9198000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 0b1b264125525..0e5de024f4589 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -dimension_values,9197000 +resolved_fields_caps,9198000 diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index ceb84e4b2a0d9..55c8b842d3db8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -53,7 +53,7 @@ protected FieldCapabilitiesResponse createTestInstance() { var indexMode = randomFrom(IndexMode.values()); responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean(), indexMode)); } - randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList()); + randomResponse = FieldCapabilitiesResponse.builder().withIndexResponses(responses).build(); return randomResponse; } @@ -88,7 +88,7 @@ protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse res ); } } - return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); + return FieldCapabilitiesResponse.builder().withFields(mutatedResponses).build(); } public void testFailureSerialization() throws IOException { @@ -144,7 +144,7 @@ public static FieldCapabilitiesResponse createResponseWithFailures() { failures.get(failures.size() - 1).addIndex(index); } } - return new FieldCapabilitiesResponse(indices, Collections.emptyMap(), failures); + return FieldCapabilitiesResponse.builder().withIndices(indices).withFailures(failures).build(); } private static FieldCapabilitiesResponse randomCCSResponse(List indexResponses) { @@ -154,7 +154,7 @@ private static FieldCapabilitiesResponse randomCCSResponse(List failureMap = List.of( new FieldCapabilitiesFailure(new String[] { "errorindex", "errorindex2" }, new IllegalArgumentException("test")) ); - return new FieldCapabilitiesResponse(new String[] { "index1", "index2", "index3", "index4" }, responses, failureMap); + return FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "index1", "index2", "index3", "index4" }) + .withFields(responses) + .withFailures(failureMap) + .build(); } public void testChunking() { diff --git a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java index 84c057d3b6a81..4a181d816451b 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java @@ -61,7 +61,12 @@ public static FieldCapabilitiesFailure parseFailure(XContentParser parser) throw .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); List indices = a[1] == null ? Collections.emptyList() : (List) a[1]; List failures = a[2] == null ? Collections.emptyList() : (List) a[2]; - return new FieldCapabilitiesResponse(indices.toArray(String[]::new), responseMap, failures); + + return FieldCapabilitiesResponse.builder() + .withIndices(indices.toArray(String[]::new)) + .withFields(responseMap) + .withFailures(failures) + .build(); } ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index d14bd8c8c8196..549a5f5714b8b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -427,7 +427,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() { } public void testGetResultMappings_DependentVariableMappingIsAbsent() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.empty(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -435,10 +435,9 @@ public void testGetResultMappings_DependentVariableMappingIsAbsent() { } public void testGetResultMappings_DependentVariableMappingHasNoTypes() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.emptyMap()) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.emptyMap())) + .build(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -459,10 +458,9 @@ public void testGetResultMappings_DependentVariableMappingIsPresent() { Map.of("type", "double") ) ); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy"))) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy")))) + .build(); Map resultMappings = new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java index 47574a6530837..74624988ff1e7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java @@ -46,8 +46,6 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -84,6 +82,8 @@ public class EsqlResolveFieldsAction extends HandledTransportAction listener) { - executeRequest(task, request, listener); + fieldCapsAction.executeRequest( + task, + request, + listener.map(resp -> new EsqlResolveFieldsResponse(resp, resp.minTransportVersion())) + ); } public void executeRequest(Task task, FieldCapabilitiesRequest request, ActionListener listener) { @@ -148,12 +154,7 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, Action if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) { // No indices at all! - listener.onResponse( - new EsqlResolveFieldsResponse( - new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()), - minTransportVersion.get() - ) - ); + listener.onResponse(new EsqlResolveFieldsResponse(FieldCapabilitiesResponse.empty(), minTransportVersion.get())); return; } @@ -335,7 +336,9 @@ private static void finishHim( ) { List failures = indexFailures.build(indexResponses.keySet()); if (indexResponses.isEmpty() == false) { - listener.onResponse(new FieldCapabilitiesResponse(new ArrayList<>(indexResponses.values()), failures)); + listener.onResponse( + FieldCapabilitiesResponse.builder().withIndexResponses(indexResponses.values()).withFailures(failures).build() + ); } else { // we have no responses at all, maybe because of errors if (indexFailures.isEmpty() == false) { @@ -348,13 +351,13 @@ private static void finishHim( failure -> failure.getException() instanceof IllegalStateException ise && ise.getCause() instanceof ElasticsearchTimeoutException )) { - listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyList(), failures)); + listener.onResponse(FieldCapabilitiesResponse.builder().withFailures(failures).build()); } else { // throw back the first exception listener.onFailure(failures.get(0).getException()); } } else { - listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyList(), Collections.emptyList())); + listener.onResponse(FieldCapabilitiesResponse.empty()); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java index 365b2b976e2f1..8ca0cc2042b8f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsResponse.java @@ -65,4 +65,12 @@ public FieldCapabilitiesResponse caps() { public TransportVersion minTransportVersion() { return minTransportVersion; } + + /** + * The minimum transport version was added to the field caps response in 9.2.1; in clusters with older nodes, + * we don't have that information and need to assume the oldest supported version. + */ + public TransportVersion requireMinTransportVersion() { + return minTransportVersion != null ? minTransportVersion : TransportVersion.minimumCompatible(); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index 0744fd126999d..f0a711a9c35c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -447,31 +447,22 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas } try (ThreadContext.StoredContext ignored = threadContext.stashWithOrigin(ClientHelper.ENRICH_ORIGIN)) { String indexName = EnrichPolicy.getBaseName(policyName); - indexResolver.resolveAsMergedMapping( - indexName, - IndexResolver.ALL_FIELDS, - null, - false, - // Disable aggregate_metric_double and dense_vector until we get version checks in planning - false, - false, - refs.acquire(indexResult -> { - if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { - EsIndex esIndex = indexResult.get(); - var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); - var resolved = new ResolvedEnrichPolicy( - p.getMatchField(), - p.getType(), - p.getEnrichFields(), - concreteIndices, - esIndex.mapping() - ); - resolvedPolices.put(policyName, resolved); - } else { - failures.put(policyName, indexResult.toString()); - } - }) - ); + indexResolver.resolveMapping(indexName, IndexResolver.ALL_FIELDS, refs.acquire(indexResult -> { + if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { + EsIndex esIndex = indexResult.get(); + var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); + var resolved = new ResolvedEnrichPolicy( + p.getMatchField(), + p.getType(), + p.getEnrichFields(), + concreteIndices, + esIndex.mapping() + ); + resolvedPolices.put(policyName, resolved); + } else { + failures.put(policyName, indexResult.toString()); + } + })); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java index 4f74561495c58..7d40f5ebe63da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java @@ -351,6 +351,31 @@ public static void initCrossClusterState( } } + public static void initCrossClusterState( + Map groupedIndices, + EsqlExecutionInfo executionInfo, + XPackLicenseState licenseState + ) { + executionInfo.clusterInfoInitializing(true); + // initialize the cluster entries in EsqlExecutionInfo before throwing the invalid license error + // so that the CCS telemetry handler can recognize that this error is CCS-related + try { + for (var entry : groupedIndices.entrySet()) { + final String clusterAlias = entry.getKey(); + executionInfo.swapCluster(clusterAlias, (k, v) -> { + assert v == null : "No cluster for " + clusterAlias + " should have been added to ExecutionInfo yet"; + return new EsqlExecutionInfo.Cluster(clusterAlias, entry.getValue(), executionInfo.shouldSkipOnFailure(clusterAlias)); + }); + } + } finally { + executionInfo.clusterInfoInitializing(false); + } + + if (executionInfo.isCrossClusterSearch() && EsqlLicenseChecker.isCcsAllowed(licenseState) == false) { + throw EsqlLicenseChecker.invalidLicenseForCcsException(licenseState); + } + } + /** * Mark cluster with a final status (success or failure). * Most metrics are set to 0 if not set yet, except for "took" which is set to the total time taken so far. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index d694588bf18c4..5ee7710594273 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -532,10 +532,8 @@ private void resolveIndicesAndAnalyze( PreAnalysisResult result, ActionListener> logicalPlanListener ) { - EsqlCCSUtils.initCrossClusterState(indicesExpressionGrouper, verifier.licenseState(), preAnalysis.indexPattern(), executionInfo); - - // The main index pattern dictates on which nodes the query can be executed, so we use the minimum transport version from this field - // caps request. + // The main index pattern dictates on which nodes the query can be executed, + // so we use the minimum transport version from this field caps request. SubscribableListener.newForked( l -> preAnalyzeMainIndicesAndRetrieveMinTransportVersion(preAnalysis, executionInfo, result, requestFilter, l) ).andThenApply(r -> { @@ -589,14 +587,9 @@ private void preAnalyzeLookupIndex( ThreadPool.Names.SEARCH_COORDINATION, ThreadPool.Names.SYSTEM_READ ); - indexResolver.resolveAsMergedMapping( + indexResolver.resolveMapping( EsqlCCSUtils.createQualifiedLookupIndexExpressionFromAvailableClusters(executionInfo, localPattern), result.wildcardJoinIndices().contains(localPattern) ? IndexResolver.ALL_FIELDS : result.fieldNames, - null, - false, - // Disable aggregate_metric_double and dense_vector until we get version checks in planning - false, - false, listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)) ); } @@ -799,7 +792,7 @@ private void preAnalyzeMainIndicesAndRetrieveMinTransportVersion( .withMinimumTransportVersion(TransportVersion.current()) ); } else { - indexResolver.resolveAsMergedMappingAndRetrieveMinimumVersion( + indexResolver.resolve( preAnalysis.indexPattern().indexPattern(), result.fieldNames, // Maybe if no indices are returned, retry without index mode and provide a clearer error message. @@ -816,9 +809,14 @@ private void preAnalyzeMainIndicesAndRetrieveMinTransportVersion( preAnalysis.supportsAggregateMetricDouble(), preAnalysis.supportsDenseVector(), listener.delegateFailureAndWrap((l, indexResolution) -> { - EsqlCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.inner().failures()); + EsqlCCSUtils.initCrossClusterState(indexResolution.groupedIndices(), executionInfo, verifier.licenseState()); + EsqlCCSUtils.updateExecutionInfoWithUnavailableClusters( + executionInfo, + indexResolution.indexResolution().failures() + ); l.onResponse( - result.withIndices(indexResolution.inner()).withMinimumTransportVersion(indexResolution.minimumVersion()) + result.withIndices(indexResolution.indexResolution()) + .withMinimumTransportVersion(indexResolution.minimumVersion()) ); }) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 49d9476126d4d..6ce24d0a451a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -19,9 +19,8 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.logging.LogManager; -import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -44,7 +43,11 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.stream.Stream; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.mapping; import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; @@ -54,7 +57,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; public class IndexResolver { - private static Logger LOGGER = LogManager.getLogger(IndexResolver.class); public static final Set ALL_FIELDS = Set.of("*"); public static final Set INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX); @@ -81,67 +83,64 @@ public IndexResolver(Client client) { this.client = client; } - /** - * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. - */ - public void resolveAsMergedMapping( - String indexWildcard, - Set fieldNames, - QueryBuilder requestFilter, - boolean includeAllDimensions, - boolean supportsAggregateMetricDouble, - boolean supportsDenseVector, - ActionListener listener - ) { - ActionListener> ignoreVersion = listener.delegateFailureAndWrap( - (l, versionedResolution) -> l.onResponse(versionedResolution.inner()) - ); - - resolveAsMergedMappingAndRetrieveMinimumVersion( - indexWildcard, - fieldNames, - requestFilter, - includeAllDimensions, - supportsAggregateMetricDouble, - supportsDenseVector, - ignoreVersion - ); - } - /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. Also retrieves the minimum transport * version available in the cluster (and remotes). */ - public void resolveAsMergedMappingAndRetrieveMinimumVersion( - String indexWildcard, + public void resolve( + String indexPattern, Set fieldNames, QueryBuilder requestFilter, boolean includeAllDimensions, boolean supportsAggregateMetricDouble, boolean supportsDenseVector, - ActionListener> listener + ActionListener listener ) { client.execute( EsqlResolveFieldsAction.TYPE, - createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions), + createFieldCapsRequest(indexPattern, fieldNames, requestFilter, includeAllDimensions), listener.delegateFailureAndWrap((l, response) -> { - TransportVersion minimumVersion = response.minTransportVersion(); - - LOGGER.debug("minimum transport version {}", minimumVersion); l.onResponse( - new Versioned<>( - mergedMappings(indexWildcard, new FieldsInfo(response.caps(), supportsAggregateMetricDouble, supportsDenseVector)), - // The minimum transport version was added to the field caps response in 9.2.1; in clusters with older nodes, - // we don't have that information and need to assume the oldest supported version. - minimumVersion == null ? TransportVersion.minimumCompatible() : minimumVersion + new IndexPatternResolution( + indexPattern, + groupedIndices(response.caps()), + mergedMappings(indexPattern, new FieldsInfo(response.caps(), supportsAggregateMetricDouble, supportsDenseVector)), + response.requireMinTransportVersion() ) ); }) ); } + public record IndexPatternResolution( + String indexPattern, + Map groupedIndices, + IndexResolution indexResolution, + TransportVersion minimumVersion + ) {} + + public void resolveMapping(String indexWildcard, Set fieldNames, ActionListener listener) { + resolve(indexWildcard, fieldNames, null, false, false, false, listener.map(IndexPatternResolution::indexResolution)); + } + public record FieldsInfo(FieldCapabilitiesResponse caps, boolean supportAggregateMetricDouble, boolean supportDenseVector) {} + // public for testing only + public static Map groupedIndices(FieldCapabilitiesResponse response) { + return Stream.concat( + Stream.of(Map.entry(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, response.getResolvedLocally())), + response.getResolvedRemotely().entrySet().stream() + ) + .flatMap( + entry -> entry.getValue() + .expressions() + .stream() + .filter(e -> e.localExpressions().indices().isEmpty() == false) + .map(e -> Map.entry(entry.getKey(), e.original())) + ) + .collect(groupingBy(Map.Entry::getKey, mapping(Map.Entry::getValue, joining(",")))); + } + // public for testing only public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fieldsInfo) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker @@ -376,6 +375,7 @@ private static FieldCapabilitiesRequest createFieldCapsRequest( req.filters("-nested"); } req.setMergeResults(false); + req.includeResolvedTo(true); return req; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 1423699ab7b45..564fb02a2326e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3152,13 +3152,14 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ) + ) + .build(), true, true ) @@ -3179,13 +3180,14 @@ public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeyword IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ) + ) + .build(), true, true ) @@ -3207,14 +3209,15 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ) + ) + .build(), true, true ) @@ -3235,13 +3238,14 @@ public void testResolveInsist_multiIndexSameMapping_fieldIsMapped() { IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) + ) + ) + .build(), true, true ) @@ -3260,15 +3264,16 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("qux", Map.of()) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), + fieldCapabilitiesIndexResponse("qux", Map.of()) + ) + ) + .build(), true, true ) @@ -3289,14 +3294,15 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa IndexResolution resolution = IndexResolver.mergedMappings( "foo, bar", new IndexResolver.FieldsInfo( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) - ), - List.of() - ), + FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), + fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) + ) + ) + .build(), true, true ) @@ -3313,10 +3319,11 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa } public void testResolveDenseVector() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))), - List.of() - ); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))) + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); @@ -3332,15 +3339,16 @@ public void testResolveDenseVector() { } public void testResolveAggregateMetricDouble() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse( - "foo", - Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse( + "foo", + Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + ) ) - ), - List.of() - ); + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); @@ -3801,7 +3809,11 @@ private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throw List idxResponses = List.of( new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); - IndexResolver.FieldsInfo caps = new IndexResolver.FieldsInfo(new FieldCapabilitiesResponse(idxResponses, List.of()), true, true); + IndexResolver.FieldsInfo caps = new IndexResolver.FieldsInfo( + FieldCapabilitiesResponse.builder().withIndexResponses(idxResponses).build(), + true, + true + ); IndexResolution resolution = IndexResolver.mergedMappings("test*", caps); var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index 9cb735b955d09..fc85cb2bf5791 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -506,9 +506,9 @@ protected void fieldCaps.put(e.getKey(), f); } var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true, IndexMode.STANDARD); - response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of()); + response = FieldCapabilitiesResponse.builder().withIndexResponses(List.of(indexResponse)).build(); } else { - response = new FieldCapabilitiesResponse(List.of(), List.of()); + response = FieldCapabilitiesResponse.empty(); } threadPool().executor(ThreadPool.Names.SEARCH_COORDINATION) .execute( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/PlanExecutorMetricsTests.java index bf8434e3c11c5..db24318314443 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/PlanExecutorMetricsTests.java @@ -150,7 +150,7 @@ public void testFailedMetric() { // simulate a valid field_caps response so we can parse and correctly analyze de query listener.onResponse( new EsqlResolveFieldsResponse( - new FieldCapabilitiesResponse(indexFieldCapabilities(indices), List.of()), + FieldCapabilitiesResponse.builder().withIndexResponses(indexFieldCapabilities(indices)).build(), TransportVersion.current() ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index fbb22c49af331..7fe7fb2150b02 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -52,7 +52,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, ) ); - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder().withIndexResponses(idxResponses).build(); // IndexResolver uses EsqlDataTypeRegistry directly IndexResolution resolution = IndexResolver.mergedMappings("idx-*", new IndexResolver.FieldsInfo(caps, true, true)); EsField f = resolution.get().mapping().get(field); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index b7646f430726a..a3ebdadb20db8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -285,14 +285,20 @@ private Map testCreateDestinationIndex(DataFrameAnalysis analysi doAnswer(callListenerOnResponse(getMappingsResponse)).when(client) .execute(eq(GetMappingsAction.INSTANCE), getMappingsRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); @@ -615,14 +621,20 @@ private Map testUpdateMappingsToDestIndex(DataFrameAnalysis anal doAnswer(callListenerOnResponse(AcknowledgedResponse.TRUE)).when(client) .execute(eq(TransportPutMappingAction.TYPE), putMappingRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index f028f39c6069f..4b9c25d71f452 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1724,7 +1724,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi } private FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(new String[] { "test" }, fieldCaps); + return FieldCapabilitiesResponse.builder().withIndices(new String[] { "test" }).withFields(fieldCaps).build(); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 0610721c04537..27a4adb910bf0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -600,7 +600,7 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps) + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build() ); } @@ -612,7 +612,7 @@ private static List separateMappings( return IndexResolver.separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, - new FieldCapabilitiesResponse(indexNames, fieldCaps), + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build(), null ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index 18af78d704646..49b58f0dde2b4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -87,16 +87,18 @@ public void testRemoveInternalFields() { } public void testExtractFieldMappings() { - FieldCapabilitiesResponse response = new FieldCapabilitiesResponse( - new String[] { "some-index" }, - Map.ofEntries( - entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), - entry( - "field-2", - Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "some-index" }) + .withFields( + Map.ofEntries( + entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), + entry( + "field-2", + Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + ) ) ) - ); + .build(); assertThat( DocumentConversionUtils.extractFieldMappings(response), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index d65428a3912de..580e3fb2bdbfa 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -287,7 +287,10 @@ protected void responseMap.put(field, singletonMap(field, createFieldCapabilities(field, type))); } - final FieldCapabilitiesResponse response = new FieldCapabilitiesResponse(fieldCapsRequest.indices(), responseMap); + final FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(fieldCapsRequest.indices()) + .withFields(responseMap) + .build(); listener.onResponse((Response) response); return; }