diff --git a/docs/changelog/136632.yaml b/docs/changelog/136632.yaml new file mode 100644 index 0000000000000..1a63e135fefcc --- /dev/null +++ b/docs/changelog/136632.yaml @@ -0,0 +1,6 @@ +pr: 136632 +summary: Field caps transport changes to return for each original expression what + it was resolved to +area: Search +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 650fad8a496ad..25fe5bdb222cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -11,8 +11,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -22,11 +25,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; @@ -34,6 +39,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { @@ -268,6 +274,138 @@ public void testReturnAllLocal() { } } + public void testResolvedToMatchingEverywhere() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertEquals( + remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + } + + public void testResolvedToMatchingLocallyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(0)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + assertEquals( + remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + assertEquals(0, remoteIndicesList.size()); + } + + public void testResolvedToMatchingRemotelyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + + FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .setIndicesOptions(options) // without ignore unavaliable would throw error + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(0)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0); + assertEquals( + remoteExpression.localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertEquals(1, remoteExpression.localExpressions().indices().size()); + assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original()); + } + public void testIncludesMinTransportVersion() { if (randomBoolean()) { assertAcked(client().admin().indices().prepareCreate("index")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index b9fcec556213a..91a990d326e54 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -13,6 +13,8 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -102,6 +104,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -913,6 +916,170 @@ public void testIndexMode() throws Exception { assertThat(actualIndexModes, equalTo(indexModes)); } + public void testResolvedExpressionWithIndexAlias() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("current", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index")); + } + + public void testResolvedExpressionWithWildcard() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("*index", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + } + + public void testResolvedExpressionWithClosedIndices() throws IOException { + // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite + assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); + ensureGreen("index1-error", "index2-error"); + + // Closed shards will result to index error because shards must be in readable state + closeShards(internalCluster(), "index1-error", "index2-error"); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error") + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + Set openIndices = Set.of("old_index", "new_index"); + Set closedIndices = Set.of("index1-error", "index2-error"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(4, resolvedLocally.expressions().size()); + for (ResolvedIndexExpression expression : expressions) { + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + if (openIndices.contains(expression.original())) { + assertThat(expression.localExpressions().indices(), containsInAnyOrder(expression.original())); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } else if (closedIndices.contains(expression.original())) { + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + } + } + + public void testResolvedExpressionWithAllIndices() { + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("_all", expression.original()); // not setting indices means _all + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } + + public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index") + .setFields("*") + .setIndicesOptions(options) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, expressions.size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("old_index", expression.original()); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + + public void testResolvedExpressionWithIndexFilter() throws InterruptedException { + assertAcked( + prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"), + prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long") + ); + + List reqs = new ArrayList<>(); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08")); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08")); + indexRandom(true, reqs); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*") + .setFields("*") + .setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01")) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response, "index-2"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-1", "index-2")); + } + + public void testNoneExpressionIndices() { + // The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices + FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get(); + + assertThat(response.getIndices().length, is(0)); + } + + public void testExclusion() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").get(); + assertIndices(response, "index-2024"); + } + + public void testExclusionWithResolvedTo() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "index-2024"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-2024", "index-2025")); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java index e1db2128143e9..1204ab001e14e 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java @@ -71,6 +71,19 @@ public void writeTo(StreamOutput out) throws IOException { * or unauthorized concrete resources. * A wildcard expression resolving to nothing is still considered a successful resolution. * The NONE result indicates that no local resolution was attempted because the expression is known to be remote-only. + * + * This distinction is needed to return either 403 (forbidden) or 404 (not found) to the user, + * and must be propagated by the linked projects to the request coordinator. + * + * CONCRETE_RESOURCE_NOT_VISIBLE: Indicates that a non-wildcard expression was resolved to nothing, + * either because the index does not exist or is closed. + * + * CONCRETE_RESOURCE_UNAUTHORIZED: Indicates that the expression could be resolved to a concrete index, + * but the requesting user is not authorized to access it. + * + * NONE: No local resolution was attempted, typically because the expression is remote-only. + * + * SUCCESS: Local index resolution was successful. */ public enum LocalIndexResolutionResult { NONE, diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java index 301db629d8d69..007db112183dd 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java @@ -73,6 +73,14 @@ public void addExpressions( ); } + /** + * Add a new resolved expression. + * @param expression the expression you want to add. + */ + public void addExpression(ResolvedIndexExpression expression) { + expressions.add(expression); + } + public void addRemoteExpressions(String original, Set remoteExpressions) { Objects.requireNonNull(original); Objects.requireNonNull(remoteExpressions); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index c4c974ad6b668..c17cae1779056 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -40,6 +40,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = TransportVersion.fromName("field_caps_add_cluster_alias"); + static final TransportVersion RESOLVED_FIELDS_CAPS = TransportVersion.fromName("resolved_fields_caps"); private String clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; @@ -58,6 +59,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen */ private transient boolean includeIndices = false; + private boolean includeResolvedTo = false; + /** * Controls whether all local indices should be returned if no remotes matched * See {@link org.elasticsearch.transport.RemoteClusterService#groupIndices} returnLocalAll argument. @@ -93,6 +96,11 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException { } else { clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; } + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + includeResolvedTo = in.readBoolean(); + } else { + includeResolvedTo = false; + } } public FieldCapabilitiesRequest() {} @@ -145,6 +153,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(FIELD_CAPS_ADD_CLUSTER_ALIAS)) { out.writeOptionalString(clusterAlias); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeBoolean(includeResolvedTo); + } } @Override @@ -223,6 +234,11 @@ public FieldCapabilitiesRequest includeIndices(boolean includeIndices) { return this; } + public FieldCapabilitiesRequest includeResolvedTo(boolean includeResolvedTo) { + this.includeResolvedTo = includeResolvedTo; + return this; + } + public FieldCapabilitiesRequest returnLocalAll(boolean returnLocalAll) { this.returnLocalAll = returnLocalAll; return this; @@ -256,6 +272,10 @@ public boolean includeIndices() { return includeIndices; } + public boolean includeResolvedTo() { + return includeResolvedTo; + } + public boolean returnLocalAll() { return returnLocalAll; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 4437895c7e08d..9e79c52ce8ae1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -63,4 +63,9 @@ public FieldCapabilitiesRequestBuilder setReturnLocalAll(boolean returnLocalAll) request().returnLocalAll(returnLocalAll); return this; } + + public FieldCapabilitiesRequestBuilder setIncludeResolvedTo(boolean resolvedTo) { + request().includeResolvedTo(resolvedTo); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index daea9b1a5fc01..8372ad5909de1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -27,6 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; /** * Response for {@link FieldCapabilitiesRequest} requests. @@ -41,6 +45,19 @@ public class FieldCapabilitiesResponse extends ActionResponse implements Chunked public static final ParseField FAILURES_FIELD = new ParseField("failures"); private final String[] indices; + + // Index expressions resolved in the context of the node that creates this response. + // If created on a linked project, "local" refers to that linked project. + // If created on the coordinating node, "local" refers to the coordinator itself. + // This data is sent from linked projects to the coordinator to inform it of each remote's local resolution state. + private final ResolvedIndexExpressions resolvedLocally; + // Remotely resolved index expressions, keyed by project alias. + // This is only populated by the coordinating node with the `resolvedLocally` data structure it receives + // back from the remotes. Used in the coordinating node for error checking, it's never sent over the wire. + // Keeping this distinction (between resolvedLocally and resolvedRemotely) further prevents project chaining + // and simplifies resolution logic, because the remoteExpressions in the resolvedLocally data structure are + // used to access data in `resolvedRemotely`. + private final transient Map resolvedRemotely; private final Map> fields; private final List failures; private final List indexResponses; @@ -51,25 +68,45 @@ public FieldCapabilitiesResponse( Map> fields, List failures ) { - this(indices, fields, Collections.emptyList(), failures, null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), failures, null); } public FieldCapabilitiesResponse(String[] indices, Map> fields) { - this(indices, fields, Collections.emptyList(), Collections.emptyList(), null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), Collections.emptyList(), null); + } + + public static FieldCapabilitiesResponse empty() { + return new FieldCapabilitiesResponse( + Strings.EMPTY_ARRAY, + null, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyList(), + null + ); } public FieldCapabilitiesResponse(List indexResponses, List failures) { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses, failures, null); + this(Strings.EMPTY_ARRAY, null, Collections.emptyMap(), Collections.emptyMap(), indexResponses, failures, null); + } + + public static FieldCapabilitiesResponse.Builder builder() { + return new FieldCapabilitiesResponse.Builder(); } private FieldCapabilitiesResponse( String[] indices, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, Map> fields, List indexResponses, List failures, TransportVersion minTransportVersion ) { this.fields = Objects.requireNonNull(fields); + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = Objects.requireNonNull(resolvedRemotely); this.indexResponses = Objects.requireNonNull(indexResponses); this.indices = indices; this.failures = failures; @@ -84,6 +121,14 @@ public FieldCapabilitiesResponse(StreamInput in) throws IOException { this.minTransportVersion = in.getTransportVersion().supports(MIN_TRANSPORT_VERSION) ? in.readOptional(TransportVersion::readVersion) : null; + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + this.resolvedLocally = in.readOptionalWriteable(ResolvedIndexExpressions::new); + } else { + this.resolvedLocally = null; + } + // when receiving a response we expect the resolved remotely to be empty. + // It's only non-empty on the coordinating node if the FC requests targets remotes. + this.resolvedRemotely = Collections.emptyMap(); } /** @@ -126,6 +171,20 @@ public List getIndexResponses() { return indexResponses; } + /** + * Locally resolved index expressions + */ + public ResolvedIndexExpressions getResolvedLocally() { + return resolvedLocally; + } + + /** + * Remotely resolved index expressions, non-empty only in the FC coordinator + */ + public Map getResolvedRemotely() { + return resolvedRemotely; + } + /** * Get the field capabilities per type for the provided {@code field}. */ @@ -145,7 +204,7 @@ public TransportVersion minTransportVersion() { * Build a new response replacing the {@link #minTransportVersion()}. */ public FieldCapabilitiesResponse withMinTransportVersion(TransportVersion newMin) { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, newMin); + return new FieldCapabilitiesResponse(indices, resolvedLocally, resolvedRemotely, fields, indexResponses, failures, newMin); } /** @@ -172,6 +231,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(MIN_TRANSPORT_VERSION)) { out.writeOptional((Writer) (o, v) -> TransportVersion.writeVersion(v, o), minTransportVersion); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeOptionalWriteable(resolvedLocally); + } } private static void writeField(StreamOutput out, Map map) throws IOException { @@ -210,6 +272,8 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; return Arrays.equals(indices, that.indices) + && Objects.equals(resolvedLocally, that.resolvedLocally) + && Objects.equals(resolvedRemotely, that.resolvedRemotely) && Objects.equals(fields, that.fields) && Objects.equals(indexResponses, that.indexResponses) && Objects.equals(failures, that.failures) @@ -218,7 +282,9 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(fields, indexResponses, failures, minTransportVersion) * 31 + Arrays.hashCode(indices); + int result = Objects.hash(resolvedLocally, resolvedRemotely, fields, indexResponses, failures, minTransportVersion); + result = 31 * result + Arrays.hashCode(indices); + return result; } @Override @@ -226,12 +292,10 @@ public String toString() { return indexResponses.isEmpty() ? Strings.toString(this) : "FieldCapabilitiesResponse{unmerged}"; } - public static Builder builder() { - return new Builder(); - } - public static class Builder { private String[] indices = Strings.EMPTY_ARRAY; + private ResolvedIndexExpressions resolvedLocally; + private Map resolvedRemotely = Collections.emptyMap(); private Map> fields = Collections.emptyMap(); private List indexResponses = Collections.emptyList(); private List failures = Collections.emptyList(); @@ -244,6 +308,24 @@ public Builder withIndices(String[] indices) { return this; } + public Builder withResolved(ResolvedIndexExpressions resolvedLocally, Map resolvedRemotely) { + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = resolvedRemotely; + return this; + } + + public Builder withResolvedRemotelyBuilder(Map resolvedRemotelyBuilder) { + this.resolvedRemotely = resolvedRemotelyBuilder.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); + return this; + } + + public Builder withResolvedLocally(ResolvedIndexExpressions resolvedLocally) { + this.resolvedLocally = resolvedLocally; + return this; + } + public Builder withFields(Map> fields) { this.fields = fields; return this; @@ -265,7 +347,15 @@ public Builder withMinTransportVersion(TransportVersion minTransportVersion) { } public FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, minTransportVersion); + return new FieldCapabilitiesResponse( + indices, + resolvedLocally, + resolvedRemotely, + fields, + indexResponses, + failures, + minTransportVersion + ); } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index ffe14bcc6af50..87f74c0704c53 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -21,15 +21,20 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.support.AbstractThreadedActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -71,6 +76,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -80,9 +86,11 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVersionCompatibility; public class TransportFieldCapabilitiesAction extends HandledTransportAction { + public static final String EXCLUSION = "-"; public static final String NAME = "indices:data/read/field_caps"; public static final ActionType TYPE = new ActionType<>(NAME); public static final RemoteClusterActionType REMOTE_TYPE = new RemoteClusterActionType<>( @@ -191,21 +199,67 @@ private void doExecuteForked( final Map remoteClusterIndices = transportService.getRemoteClusterService() .groupIndices(request.indicesOptions(), request.indices(), request.returnLocalAll()); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - final String[] concreteIndices = localIndices != null - ? indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices) - : Strings.EMPTY_ARRAY; - - if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) { - listener.onResponse( - linkedRequestExecutor.wrapPrimary( - FieldCapabilitiesResponse.builder().withMinTransportVersion(minTransportVersion.get()).build() - ) - ); + + final String[] concreteLocalIndices; + final List resolvedLocallyList; + if (request.getResolvedIndexExpressions() != null) { + // in CPS the Security Action Filter would populate resolvedExpressions for the local project + // thus we can get the concreteLocalIndices based on the resolvedLocallyList + resolvedLocallyList = request.getResolvedIndexExpressions().expressions(); + concreteLocalIndices = resolvedLocallyList.stream() + .map(r -> r.localExpressions().indices()) + .flatMap(Set::stream) + .distinct() + .toArray(String[]::new); + } else { + // In CCS/Local only search we have to populate resolvedLocallyList one by one for each localIndices.indices() + // only if the request is includeResolvedTo() + resolvedLocallyList = new ArrayList<>(); + if (localIndices == null) { + // in the case we have one or more remote indices but no local we don't expand to all local indices + // in this case resolvedLocallyList will remain empty + concreteLocalIndices = Strings.EMPTY_ARRAY; + } else { + concreteLocalIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices); + if (request.includeResolvedTo()) { + ProjectMetadata projectMetadata = projectState.metadata(); + IndicesOptions indicesOptions = localIndices.indicesOptions(); + String[] localIndexNames = localIndices.indices(); + if (localIndexNames.length == 0) { + // Empty indices array means match all + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(projectMetadata, indicesOptions); + resolvedLocallyList.add(createResolvedIndexExpression(Metadata.ALL, concreteIndexNames)); + } else if (false == IndexNameExpressionResolver.isNoneExpression(localIndexNames)) { + // if it's neither match all nor match none, but we want to include resolutions we loop for all the indicesNames + for (String localIndexName : localIndexNames) { + if (false == localIndexName.startsWith(EXCLUSION)) { + // we populate resolvedLocally iff is not an exclusion + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames( + projectMetadata, + indicesOptions, + localIndices.includeDataStreams(), + localIndexName + ); + resolvedLocallyList.add(createResolvedIndexExpression(localIndexName, concreteIndexNames)); + } + } + } + } + + } + } + + if (concreteLocalIndices.length == 0 && remoteClusterIndices.isEmpty()) { + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder(); + responseBuilder.withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo()) { + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(resolvedLocallyList)); + } + listener.onResponse(linkedRequestExecutor.wrapPrimary(responseBuilder.build())); return; } - checkIndexBlocks(projectState, concreteIndices); + checkIndexBlocks(projectState, concreteLocalIndices); final FailureCollector indexFailures = new FailureCollector(); final Map indexResponses = new HashMap<>(); // This map is used to share the index response for indices which have the same index mapping hash to reduce the memory usage. @@ -216,6 +270,10 @@ private void doExecuteForked( indexResponses.clear(); indexMappingHashToResponses.clear(); }; + Map resolvedRemotely = new ConcurrentHashMap<>(); + for (String clusterAlias : remoteClusterIndices.keySet()) { + resolvedRemotely.put(clusterAlias, ResolvedIndexExpressions.builder()); + } final Consumer handleIndexResponse = resp -> { if (fieldCapTask.isCancelled()) { releaseResourcesOnCancel.run(); @@ -282,6 +340,8 @@ private void doExecuteForked( fieldCapTask, indexResponses, indexFailures, + resolvedLocallyList, + resolvedRemotely, minTransportVersion, listener.map(linkedRequestExecutor::wrapPrimary) ); @@ -297,7 +357,7 @@ private void doExecuteForked( request, localIndices, nowInMillis, - concreteIndices, + concreteLocalIndices, singleThreadedExecutor, handleIndexResponse, handleIndexFailure, @@ -312,6 +372,21 @@ private void doExecuteForked( OriginalIndices originalIndices = remoteIndices.getValue(); FieldCapabilitiesRequest remoteRequest = prepareRemoteRequest(clusterAlias, request, originalIndices, nowInMillis); ActionListener remoteListener = ActionListener.wrap(response -> { + + if (request.includeResolvedTo() && response.getResolvedLocally() != null) { + ResolvedIndexExpressions resolvedOnRemoteProject = response.getResolvedLocally(); + // for bwc we need to check that resolvedOnRemoteProject Exists in the response + if (resolvedOnRemoteProject != null) { + for (ResolvedIndexExpression remoteResolvedExpression : resolvedOnRemoteProject.expressions()) { + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(remoteResolvedExpression); + return v; + }); + } + } + + } + for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()); handleIndexResponse.accept( @@ -328,6 +403,21 @@ private void doExecuteForked( Exception ex = failure.getException(); for (String index : failure.getIndices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } } minTransportVersion.accumulateAndGet(response.minTransportVersion(), (lhs, rhs) -> { @@ -339,6 +429,21 @@ private void doExecuteForked( }, ex -> { for (String index : originalIndices.indices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } }); @@ -377,6 +482,20 @@ private void doExecuteForked( } } + private static ResolvedIndexExpression createResolvedIndexExpression(String original, String[] concreteIndexNames) { + boolean isWildcard = Regex.isSimpleMatchPattern(original); + // if it is a wildcard we consider it successful even if it didn't resolve to any concrete index + ResolvedIndexExpression.LocalIndexResolutionResult resolutionResult = concreteIndexNames.length > 0 || isWildcard + ? ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + : ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE; + + return new ResolvedIndexExpression( + original, + new ResolvedIndexExpression.LocalExpressions(Set.of(concreteIndexNames), resolutionResult, null), + Collections.emptySet() + ); + } + private Executor buildSingleThreadedExecutor() { final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner("field_caps", 1, searchCoordinationExecutor); return r -> throttledTaskRunner.enqueueTask(new ActionListener<>() { @@ -433,19 +552,27 @@ private static void mergeIndexResponses( CancellableTask task, Map indexResponses, FailureCollector indexFailures, + List resolvedLocallyList, + Map resolvedRemotely, AtomicReference minTransportVersion, ActionListener listener ) { + ResolvedIndexExpressions resolvedLocally = new ResolvedIndexExpressions(resolvedLocallyList); List failures = indexFailures.build(indexResponses.keySet()); if (indexResponses.isEmpty() == false) { if (request.isMergeResults()) { - ActionListener.completeWith(listener, () -> merge(indexResponses, task, request, failures, minTransportVersion)); + ActionListener.completeWith( + listener, + () -> merge(indexResponses, resolvedLocally, resolvedRemotely, task, request, failures, minTransportVersion) + ); } else { listener.onResponse( FieldCapabilitiesResponse.builder() .withIndexResponses(new ArrayList<>(indexResponses.values())) - .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) .withMinTransportVersion(minTransportVersion.get()) + .withFailures(failures) .build() ); } @@ -460,7 +587,12 @@ private static void mergeIndexResponses( && ise.getCause() instanceof ElasticsearchTimeoutException )) { listener.onResponse( - FieldCapabilitiesResponse.builder().withFailures(failures).withMinTransportVersion(minTransportVersion.get()).build() + FieldCapabilitiesResponse.builder() + .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withMinTransportVersion(minTransportVersion.get()) + .build() ); } else { // throw back the first exception @@ -489,6 +621,7 @@ private static FieldCapabilitiesRequest prepareRemoteRequest( remoteRequest.indexFilter(request.indexFilter()); remoteRequest.nowInMillis(nowInMillis); remoteRequest.includeEmptyFields(request.includeEmptyFields()); + remoteRequest.includeResolvedTo(request.includeResolvedTo()); return remoteRequest; } @@ -500,6 +633,8 @@ private static boolean hasSameMappingHash(FieldCapabilitiesIndexResponse r1, Fie private static FieldCapabilitiesResponse merge( Map indexResponsesMap, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, CancellableTask task, FieldCapabilitiesRequest request, List failures, @@ -533,6 +668,22 @@ private static FieldCapabilitiesResponse merge( collectFields(fieldsBuilder, fields, request.includeIndices()); } + List failedIndices = failures.stream().flatMap(f -> Arrays.stream(f.getIndices())).toList(); + List collect = resolvedLocally.expressions().stream().map(expression -> { + if (failedIndices.contains(expression.original())) { + return new ResolvedIndexExpression( + expression.original(), + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + expression.remoteExpressions() + ); + } + return expression; + }).toList(); + // The merge method is only called on the primary coordinator for cross-cluster field caps, so we // log relevant "5xx" errors that occurred in this 2xx response to ensure they are only logged once. // These failures have already been deduplicated, before this method was called. @@ -544,12 +695,17 @@ private static FieldCapabilitiesResponse merge( ); } } - return FieldCapabilitiesResponse.builder() + + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder() .withIndices(indices) .withFields(Collections.unmodifiableMap(fields)) .withFailures(failures) - .withMinTransportVersion(minTransportVersion.get()) - .build(); + .withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo() && minTransportVersion.get().supports(RESOLVED_FIELDS_CAPS)) { + // add resolution to response iff includeResolvedTo and all the nodes in the cluster supports it + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(collect)).withResolvedRemotelyBuilder(resolvedRemotely); + } + return responseBuilder.build(); } private static boolean shouldLogException(Exception e) { diff --git a/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv new file mode 100644 index 0000000000000..4629e277c153b --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv @@ -0,0 +1 @@ +9212000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index a70b776735734..ce5e1c85f99fd 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -add_sample_method_downsample_dlm,9211000 +resolved_fields_caps,9212000 diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index ceb84e4b2a0d9..55c8b842d3db8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -53,7 +53,7 @@ protected FieldCapabilitiesResponse createTestInstance() { var indexMode = randomFrom(IndexMode.values()); responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean(), indexMode)); } - randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList()); + randomResponse = FieldCapabilitiesResponse.builder().withIndexResponses(responses).build(); return randomResponse; } @@ -88,7 +88,7 @@ protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse res ); } } - return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); + return FieldCapabilitiesResponse.builder().withFields(mutatedResponses).build(); } public void testFailureSerialization() throws IOException { @@ -144,7 +144,7 @@ public static FieldCapabilitiesResponse createResponseWithFailures() { failures.get(failures.size() - 1).addIndex(index); } } - return new FieldCapabilitiesResponse(indices, Collections.emptyMap(), failures); + return FieldCapabilitiesResponse.builder().withIndices(indices).withFailures(failures).build(); } private static FieldCapabilitiesResponse randomCCSResponse(List indexResponses) { @@ -154,7 +154,7 @@ private static FieldCapabilitiesResponse randomCCSResponse(List failureMap = List.of( new FieldCapabilitiesFailure(new String[] { "errorindex", "errorindex2" }, new IllegalArgumentException("test")) ); - return new FieldCapabilitiesResponse(new String[] { "index1", "index2", "index3", "index4" }, responses, failureMap); + return FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "index1", "index2", "index3", "index4" }) + .withFields(responses) + .withFailures(failureMap) + .build(); } public void testChunking() { diff --git a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java index 84c057d3b6a81..4a181d816451b 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java @@ -61,7 +61,12 @@ public static FieldCapabilitiesFailure parseFailure(XContentParser parser) throw .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); List indices = a[1] == null ? Collections.emptyList() : (List) a[1]; List failures = a[2] == null ? Collections.emptyList() : (List) a[2]; - return new FieldCapabilitiesResponse(indices.toArray(String[]::new), responseMap, failures); + + return FieldCapabilitiesResponse.builder() + .withIndices(indices.toArray(String[]::new)) + .withFields(responseMap) + .withFailures(failures) + .build(); } ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index d14bd8c8c8196..549a5f5714b8b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -427,7 +427,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() { } public void testGetResultMappings_DependentVariableMappingIsAbsent() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.empty(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -435,10 +435,9 @@ public void testGetResultMappings_DependentVariableMappingIsAbsent() { } public void testGetResultMappings_DependentVariableMappingHasNoTypes() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.emptyMap()) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.emptyMap())) + .build(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -459,10 +458,9 @@ public void testGetResultMappings_DependentVariableMappingIsPresent() { Map.of("type", "double") ) ); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy"))) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy")))) + .build(); Map resultMappings = new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e31bfdd5364b2..75fc686cf5d0d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3317,10 +3317,11 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa } public void testResolveDenseVector() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))), - List.of() - ); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))) + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", @@ -3342,15 +3343,16 @@ public void testResolveDenseVector() { } public void testResolveAggregateMetricDouble() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse( - "foo", - Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse( + "foo", + Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + ) ) - ), - List.of() - ); + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index 8ea5ecf231407..54e100e962c4b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -506,9 +506,9 @@ protected void fieldCaps.put(e.getKey(), f); } var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true, IndexMode.STANDARD); - response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of()); + response = FieldCapabilitiesResponse.builder().withIndexResponses(List.of(indexResponse)).build(); } else { - response = new FieldCapabilitiesResponse(List.of(), List.of()); + response = FieldCapabilitiesResponse.empty(); } threadPool().executor(ThreadPool.Names.SEARCH_COORDINATION) .execute(ActionRunnable.supply(listener, () -> (Response) new EsqlResolveFieldsResponse(response))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index c201f544372db..eda583457243c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -52,7 +52,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, ) ); - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder().withIndexResponses(idxResponses).build(); // IndexResolver uses EsqlDataTypeRegistry directly IndexResolution resolution = IndexResolver.mergedMappings( "idx-*", diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index b7646f430726a..a3ebdadb20db8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -285,14 +285,20 @@ private Map testCreateDestinationIndex(DataFrameAnalysis analysi doAnswer(callListenerOnResponse(getMappingsResponse)).when(client) .execute(eq(GetMappingsAction.INSTANCE), getMappingsRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); @@ -615,14 +621,20 @@ private Map testUpdateMappingsToDestIndex(DataFrameAnalysis anal doAnswer(callListenerOnResponse(AcknowledgedResponse.TRUE)).when(client) .execute(eq(TransportPutMappingAction.TYPE), putMappingRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index f028f39c6069f..4b9c25d71f452 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1724,7 +1724,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi } private FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(new String[] { "test" }, fieldCaps); + return FieldCapabilitiesResponse.builder().withIndices(new String[] { "test" }).withFields(fieldCaps).build(); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 0610721c04537..27a4adb910bf0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -600,7 +600,7 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps) + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build() ); } @@ -612,7 +612,7 @@ private static List separateMappings( return IndexResolver.separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, - new FieldCapabilitiesResponse(indexNames, fieldCaps), + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build(), null ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index 18af78d704646..49b58f0dde2b4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -87,16 +87,18 @@ public void testRemoveInternalFields() { } public void testExtractFieldMappings() { - FieldCapabilitiesResponse response = new FieldCapabilitiesResponse( - new String[] { "some-index" }, - Map.ofEntries( - entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), - entry( - "field-2", - Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "some-index" }) + .withFields( + Map.ofEntries( + entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), + entry( + "field-2", + Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + ) ) ) - ); + .build(); assertThat( DocumentConversionUtils.extractFieldMappings(response), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index d65428a3912de..580e3fb2bdbfa 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -287,7 +287,10 @@ protected void responseMap.put(field, singletonMap(field, createFieldCapabilities(field, type))); } - final FieldCapabilitiesResponse response = new FieldCapabilitiesResponse(fieldCapsRequest.indices(), responseMap); + final FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(fieldCapsRequest.indices()) + .withFields(responseMap) + .build(); listener.onResponse((Response) response); return; }