From c2feb8940417a3af8486e9c4ec71dbe67052f350 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 3 Sep 2025 18:10:08 +0300 Subject: [PATCH 1/9] Prevent field_caps from failing when can-match fails `FieldCapabilitiesFetcher` performs a can-match in order to quickly return an empty response if no shard can match. However, if can-match fails for some reason, it can cause the field capabilities request to fail. An example of that is when a semantic query is used as filter. can-match will fail as it won't be able to expand the inference results of the query. In cases like that, it makes no sense to fail the field capabilities request. Instead, we should treat can-match as returning `true` to proceed. This change does that by following suit with other callers of can-match. Fixes #116106 --- .../fieldcaps/FieldCapabilitiesFetcher.java | 9 ++- .../xpack/inference/InferenceFeatures.java | 1 + .../mapper/SemanticTextFieldMapper.java | 1 + .../100_semantic_text_field_caps.yml | 56 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index aee7858d84b13..3cb759250802d 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -249,12 +249,17 @@ private static boolean canMatchShard( QueryBuilder indexFilter, long nowInMillis, SearchExecutionContext searchExecutionContext - ) throws IOException { + ) { assert alwaysMatches(indexFilter) == false : "should not be called for always matching [" + indexFilter + "]"; assert nowInMillis != 0L; ShardSearchRequest searchRequest = new ShardSearchRequest(shardId, nowInMillis, AliasFilter.EMPTY); searchRequest.source(new SearchSourceBuilder().query(indexFilter)); - return SearchService.queryStillMatchesAfterRewrite(searchRequest, searchExecutionContext); + try { + return SearchService.queryStillMatchesAfterRewrite(searchRequest, searchExecutionContext); + } catch (Exception e) { + // treat as if shard is still a potential match + return true; + } } private static boolean alwaysMatches(QueryBuilder indexFilter) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index d5f8cebba3167..3cbccd5be1f5d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -59,6 +59,7 @@ public Set getTestFeatures() { SemanticTextFieldMapper.SEMANTIC_TEXT_DELETE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX, + SemanticTextFieldMapper.SEMANTIC_TEXT_FILTER_FIELD_CAPS_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS, SEMANTIC_TEXT_HIGHLIGHTER, SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 5b35c0384ce99..151808776373b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -136,6 +136,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final NodeFeature SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX = new NodeFeature( "semantic_text.always_emit_inference_id_fix" ); + public static final NodeFeature SEMANTIC_TEXT_FILTER_FIELD_CAPS_FIX = new NodeFeature("semantic_text.filter_field_caps_fix"); public static final NodeFeature SEMANTIC_TEXT_HANDLE_EMPTY_INPUT = new NodeFeature("semantic_text.handle_empty_input"); public static final NodeFeature SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS = new NodeFeature("semantic_text.skip_inference_fields"); public static final NodeFeature SEMANTIC_TEXT_BIT_VECTOR_SUPPORT = new NodeFeature("semantic_text.bit_vector_support"); diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml new file mode 100644 index 0000000000000..88cf75e3b5431 --- /dev/null +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml @@ -0,0 +1,56 @@ +setup: + - requires: + cluster_features: "semantic_text.filter_field_caps_fix" + reason: "fixed bug with semantic query filtering in field_caps (#116106)" + + - do: + inference.put: + task_type: sparse_embedding + inference_id: sparse-inference-id + body: > + { + "service": "test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + semantic_text_field: + type: semantic_text + inference_id: sparse-inference-id + std_text_field: + type: text + + - do: + index: + index: test-index + id: doc_1 + body: + semantic_text_field: "This is a story about a cat and a dog." + std_text_field: "some text" + refresh: true + +--- +"Field caps with semantic_text filter does not fail": + - do: + field_caps: + index: test-index + fields: "*" + body: + index_filter: + semantic: + field: "semantic_text_field" + query: "test" + + - match: { indices: [ "test-index" ] } + - match: { fields.semantic_text_field.text.searchable: true } + - match: { fields.std_text_field.text.searchable: true } From 17cd702269549b38df90461780a12a0124c9f35e Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 4 Sep 2025 16:22:34 +0300 Subject: [PATCH 2/9] Move NodeFeature to SemanticQueryBuilder --- .../org/elasticsearch/xpack/inference/InferenceFeatures.java | 4 ++-- .../xpack/inference/mapper/SemanticTextFieldMapper.java | 1 - .../xpack/inference/queries/SemanticQueryBuilder.java | 1 + .../test/inference/100_semantic_text_field_caps.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 3cbccd5be1f5d..f6b5a8760aed0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -59,7 +59,6 @@ public Set getTestFeatures() { SemanticTextFieldMapper.SEMANTIC_TEXT_DELETE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX, - SemanticTextFieldMapper.SEMANTIC_TEXT_FILTER_FIELD_CAPS_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS, SEMANTIC_TEXT_HIGHLIGHTER, SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, @@ -85,7 +84,8 @@ public Set getTestFeatures() { SEMANTIC_TEXT_HIGHLIGHTING_FLAT, SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS, SEMANTIC_TEXT_FIELDS_CHUNKS_FORMAT, - SemanticQueryBuilder.SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS + SemanticQueryBuilder.SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS, + SemanticQueryBuilder.SEMANTIC_QUERY_FILTER_FIELD_CAPS_FIX ) ); if (RERANK_SNIPPETS.isEnabled()) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 151808776373b..5b35c0384ce99 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -136,7 +136,6 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final NodeFeature SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX = new NodeFeature( "semantic_text.always_emit_inference_id_fix" ); - public static final NodeFeature SEMANTIC_TEXT_FILTER_FIELD_CAPS_FIX = new NodeFeature("semantic_text.filter_field_caps_fix"); public static final NodeFeature SEMANTIC_TEXT_HANDLE_EMPTY_INPUT = new NodeFeature("semantic_text.handle_empty_input"); public static final NodeFeature SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS = new NodeFeature("semantic_text.skip_inference_fields"); public static final NodeFeature SEMANTIC_TEXT_BIT_VECTOR_SUPPORT = new NodeFeature("semantic_text.bit_vector_support"); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java index 71eb3494aa8ce..cf90c3e93fce8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java @@ -56,6 +56,7 @@ public class SemanticQueryBuilder extends AbstractQueryBuilder Date: Thu, 4 Sep 2025 16:42:22 +0300 Subject: [PATCH 3/9] Move YML test next to other similar tests --- .../100_semantic_text_field_caps.yml | 56 ------------------- .../10_semantic_text_field_mapping.yml | 24 ++++++++ 2 files changed, 24 insertions(+), 56 deletions(-) delete mode 100644 x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml deleted file mode 100644 index 4344714daafb3..0000000000000 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/100_semantic_text_field_caps.yml +++ /dev/null @@ -1,56 +0,0 @@ -setup: - - requires: - cluster_features: "semantic_query.filter_field_caps_fix" - reason: "fixed bug with semantic query filtering in field_caps (#116106)" - - - do: - inference.put: - task_type: sparse_embedding - inference_id: sparse-inference-id - body: > - { - "service": "test_service", - "service_settings": { - "model": "my_model", - "api_key": "abc64" - }, - "task_settings": { - } - } - - - do: - indices.create: - index: test-index - body: - mappings: - properties: - semantic_text_field: - type: semantic_text - inference_id: sparse-inference-id - std_text_field: - type: text - - - do: - index: - index: test-index - id: doc_1 - body: - semantic_text_field: "This is a story about a cat and a dog." - std_text_field: "some text" - refresh: true - ---- -"Field caps with semantic_text filter does not fail": - - do: - field_caps: - index: test-index - fields: "*" - body: - index_filter: - semantic: - field: "semantic_text_field" - query: "test" - - - match: { indices: [ "test-index" ] } - - match: { fields.semantic_text_field.text.searchable: true } - - match: { fields.std_text_field.text.searchable: true } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml index 4e2f5005932f5..e7e5b04669a6f 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml @@ -548,6 +548,30 @@ setup: - not_exists: fields.dense_field.inference.chunks - not_exists: fields.dense_field.inference +--- +"Field caps with semantic query filter does not fail": + # We need at least one document present to exercise can-match phase + - do: + index: + index: test-index + id: doc_1 + body: + sparse_field: "This is a story about a cat and a dog." + refresh: true + + - do: + field_caps: + index: test-index + fields: "*" + body: + index_filter: + semantic: + field: "sparse_field" + query: "test" + + - match: { indices: [ "test-index" ] } + - match: { fields.sparse_field.text.searchable: true } + --- "Users can set dense vector index options and index documents using those options": - requires: From b2f31b3f1f394d85a293e253cabbfc1b6e61ab9c Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 4 Sep 2025 16:46:07 +0300 Subject: [PATCH 4/9] Also add test in BWC tests --- .../10_semantic_text_field_mapping_bwc.yml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml index 7cc577162e13f..0827865b4997c 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml @@ -447,6 +447,31 @@ setup: - not_exists: fields.dense_field.inference.chunks.text - not_exists: fields.dense_field.inference.chunks - not_exists: fields.dense_field.inference + +--- +"Field caps with semantic query filter does not fail": + # We need at least one document present to exercise can-match phase + - do: + index: + index: test-index + id: doc_1 + body: + sparse_field: "This is a story about a cat and a dog." + refresh: true + + - do: + field_caps: + index: test-index + fields: "*" + body: + index_filter: + semantic: + field: "sparse_field" + query: "test" + + - match: { indices: [ "test-index" ] } + - match: { fields.sparse_field.text.searchable: true } + --- "Users can set dense vector index options and index documents using those options": - requires: From c20d9e8763b50ed1878fe78f58cd389602872462 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 4 Sep 2025 16:46:57 +0300 Subject: [PATCH 5/9] Remove filter from test name --- .../test/inference/10_semantic_text_field_mapping.yml | 2 +- .../test/inference/10_semantic_text_field_mapping_bwc.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml index e7e5b04669a6f..74a84a07a4c69 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml @@ -549,7 +549,7 @@ setup: - not_exists: fields.dense_field.inference --- -"Field caps with semantic query filter does not fail": +"Field caps with semantic query does not fail": # We need at least one document present to exercise can-match phase - do: index: diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml index 0827865b4997c..144185675388f 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml @@ -449,7 +449,7 @@ setup: - not_exists: fields.dense_field.inference --- -"Field caps with semantic query filter does not fail": +"Field caps with semantic query does not fail": # We need at least one document present to exercise can-match phase - do: index: From 6ab512d3f2afaeb0bc744f8316f5779eabe0ee9e Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 5 Sep 2025 11:47:05 +0300 Subject: [PATCH 6/9] Fix field caps integration tests --- .../fieldcaps/CCSFieldCapabilitiesIT.java | 36 ++++----- .../search/fieldcaps/FieldCapabilitiesIT.java | 74 +++++++------------ 2 files changed, 38 insertions(+), 72 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index e727f5fdfced4..5d292de5fea32 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -14,15 +14,12 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT.ExceptionOnRewriteQueryBuilder; -import org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT.ExceptionOnRewriteQueryPlugin; +import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.transport.RemoteTransportException; -import java.util.ArrayList; +import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -46,14 +43,7 @@ protected boolean reuseClusters() { return false; } - @Override - protected Collection> nodePlugins(String clusterAlias) { - final List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); - plugins.add(ExceptionOnRewriteQueryPlugin.class); - return plugins; - } - - public void testFailuresFromRemote() { + public void testFailuresFromRemote() throws IOException { Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).build(); final Client localClient = client(LOCAL_CLUSTER); final Client remoteClient = client("remote_cluster"); @@ -71,10 +61,11 @@ public void testFailuresFromRemote() { FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "remote_cluster:*").setFields("*").get(); assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder(localIndex, "remote_cluster:" + remoteErrorIndex)); - // adding an index filter so remote call should fail + // Closed shards will result to index error because shards must be in readable state + FieldCapabilitiesIT.closeShards(cluster("remote_cluster"), remoteErrorIndex); + response = client().prepareFieldCaps("*", "remote_cluster:*") .setFields("*") - .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) .get(); assertThat(response.getIndices()[0], equalTo(localIndex)); assertThat(response.getFailedIndicesCount(), equalTo(1)); @@ -86,15 +77,15 @@ public void testFailuresFromRemote() { Exception ex = failure.getException(); assertEquals(RemoteTransportException.class, ex.getClass()); Throwable cause = ExceptionsHelper.unwrapCause(ex); - assertEquals(IllegalArgumentException.class, cause.getClass()); - assertEquals("I throw because I choose to.", cause.getMessage()); + assertEquals(IllegalIndexShardStateException.class, cause.getClass()); + assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", cause.getMessage()); // if we only query the remote we should get back an exception only ex = expectThrows( - IllegalArgumentException.class, - client().prepareFieldCaps("remote_cluster:*").setFields("*").setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + IllegalIndexShardStateException.class, + client().prepareFieldCaps("remote_cluster:*").setFields("*") ); - assertEquals("I throw because I choose to.", ex.getMessage()); + assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", ex.getMessage()); // add an index that doesn't fail to the remote assertAcked(remoteClient.admin().indices().prepareCreate("okay_remote_index")); @@ -103,7 +94,6 @@ public void testFailuresFromRemote() { response = client().prepareFieldCaps("*", "remote_cluster:*") .setFields("*") - .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) .get(); assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder(localIndex, "remote_cluster:okay_remote_index")); assertThat(response.getFailedIndicesCount(), equalTo(1)); @@ -113,8 +103,8 @@ public void testFailuresFromRemote() { .findFirst() .get(); ex = failure.getException(); - assertEquals(IllegalArgumentException.class, ex.getClass()); - assertEquals("I throw because I choose to.", ex.getMessage()); + assertEquals(IllegalIndexShardStateException.class, ex.getClass()); + assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", ex.getMessage()); } public void testFailedToConnectToRemoteCluster() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 05b6bf1f9166d..bad1a705c4d78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -47,7 +47,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexClosedException; @@ -59,6 +59,7 @@ import org.elasticsearch.search.DummyQueryBuilder; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockLog; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; @@ -78,6 +79,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -87,7 +89,6 @@ import java.util.function.Predicate; import java.util.stream.IntStream; -import static java.util.Collections.singletonList; import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener; import static org.elasticsearch.index.shard.IndexShardTestCase.closeShardNoCheck; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -193,7 +194,7 @@ public void setUp() throws Exception { @Override protected Collection> nodePlugins() { - return List.of(TestMapperPlugin.class, ExceptionOnRewriteQueryPlugin.class, BlockingOnRewriteQueryPlugin.class); + return List.of(TestMapperPlugin.class, BlockingOnRewriteQueryPlugin.class); } @Override @@ -445,30 +446,33 @@ public void testFieldMetricsAndDimensions() { assertThat(response.get().get("some_dimension").get("keyword").nonDimensionIndices(), array(equalTo("new_index"))); } - public void testFailures() throws InterruptedException { + public void testFailures() throws IOException { // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); ensureGreen("index1-error", "index2-error"); - FieldCapabilitiesResponse response = client().prepareFieldCaps() + + // Closed shards will result to index error because shards must be in readable state + closeShards(internalCluster(), "index1-error", "index2-error"); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error") .setFields("*") - .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) .get(); assertEquals(1, response.getFailures().size()); assertEquals(2, response.getFailedIndicesCount()); assertThat(response.getFailures().get(0).getIndices(), arrayContainingInAnyOrder("index1-error", "index2-error")); Exception failure = response.getFailures().get(0).getException(); - assertEquals(IllegalArgumentException.class, failure.getClass()); - assertEquals("I throw because I choose to.", failure.getMessage()); + assertEquals(IllegalIndexShardStateException.class, failure.getClass()); + assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", failure.getMessage()); // the "indices" section should not include failed ones assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder("old_index", "new_index")); // if all requested indices failed, we fail the request by throwing the exception - IllegalArgumentException ex = expectThrows( - IllegalArgumentException.class, - client().prepareFieldCaps("index1-error", "index2-error").setFields("*").setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + IllegalIndexShardStateException ex = expectThrows( + IllegalIndexShardStateException.class, + client().prepareFieldCaps("index1-error", "index2-error").setFields("*") ); - assertEquals("I throw because I choose to.", ex.getMessage()); + assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", ex.getMessage()); } private void populateTimeRangeIndices() throws Exception { @@ -913,45 +917,17 @@ private void assertIndices(FieldCapabilitiesResponse response, String... indices assertArrayEquals(indices, response.getIndices()); } - /** - * Adds an "exception" query that throws on rewrite if the index name contains the string "error" - */ - public static class ExceptionOnRewriteQueryPlugin extends Plugin implements SearchPlugin { - - public ExceptionOnRewriteQueryPlugin() {} - - @Override - public List> getQueries() { - return singletonList( - new QuerySpec<>("exception", ExceptionOnRewriteQueryBuilder::new, p -> new ExceptionOnRewriteQueryBuilder()) - ); - } - } - - static class ExceptionOnRewriteQueryBuilder extends DummyQueryBuilder { - - public static final String NAME = "exception"; - - ExceptionOnRewriteQueryBuilder() {} - - ExceptionOnRewriteQueryBuilder(StreamInput in) throws IOException { - super(in); - } - - @Override - protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - SearchExecutionContext searchExecutionContext = queryRewriteContext.convertToSearchExecutionContext(); - if (searchExecutionContext != null) { - if (searchExecutionContext.indexMatches("*error*")) { - throw new IllegalArgumentException("I throw because I choose to."); + static void closeShards(InternalTestCluster cluster, String... indices) throws IOException { + final Set indicesToClose = Set.of(indices); + for (String node :cluster.getNodeNames()) { + final IndicesService indicesService = cluster.getInstance(IndicesService.class, node); + for (IndexService indexService : indicesService) { + if (indicesToClose.contains(indexService.getMetadata().getIndex().getName())) { + for (IndexShard indexShard : indexService) { + closeShardNoCheck(indexShard); + } } } - return this; - } - - @Override - public String getWriteableName() { - return NAME; } } From fae7738928ee9911ede8686ab47325e24c38ea63 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 5 Sep 2025 08:56:01 +0000 Subject: [PATCH 7/9] [CI] Auto commit changes from spotless --- .../fieldcaps/CCSFieldCapabilitiesIT.java | 18 +++++++----------- .../search/fieldcaps/FieldCapabilitiesIT.java | 7 +++++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 5d292de5fea32..240a13bc601b8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -64,9 +64,7 @@ public void testFailuresFromRemote() throws IOException { // Closed shards will result to index error because shards must be in readable state FieldCapabilitiesIT.closeShards(cluster("remote_cluster"), remoteErrorIndex); - response = client().prepareFieldCaps("*", "remote_cluster:*") - .setFields("*") - .get(); + response = client().prepareFieldCaps("*", "remote_cluster:*").setFields("*").get(); assertThat(response.getIndices()[0], equalTo(localIndex)); assertThat(response.getFailedIndicesCount(), equalTo(1)); FieldCapabilitiesFailure failure = response.getFailures() @@ -78,13 +76,13 @@ public void testFailuresFromRemote() throws IOException { assertEquals(RemoteTransportException.class, ex.getClass()); Throwable cause = ExceptionsHelper.unwrapCause(ex); assertEquals(IllegalIndexShardStateException.class, cause.getClass()); - assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", cause.getMessage()); + assertEquals( + "CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", + cause.getMessage() + ); // if we only query the remote we should get back an exception only - ex = expectThrows( - IllegalIndexShardStateException.class, - client().prepareFieldCaps("remote_cluster:*").setFields("*") - ); + ex = expectThrows(IllegalIndexShardStateException.class, client().prepareFieldCaps("remote_cluster:*").setFields("*")); assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", ex.getMessage()); // add an index that doesn't fail to the remote @@ -92,9 +90,7 @@ public void testFailuresFromRemote() throws IOException { remoteClient.prepareIndex("okay_remote_index").setId("2").setSource("foo", "bar").get(); remoteClient.admin().indices().prepareRefresh("okay_remote_index").get(); - response = client().prepareFieldCaps("*", "remote_cluster:*") - .setFields("*") - .get(); + response = client().prepareFieldCaps("*", "remote_cluster:*").setFields("*").get(); assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder(localIndex, "remote_cluster:okay_remote_index")); assertThat(response.getFailedIndicesCount(), equalTo(1)); failure = response.getFailures() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index bad1a705c4d78..b9fcec556213a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -462,7 +462,10 @@ public void testFailures() throws IOException { assertThat(response.getFailures().get(0).getIndices(), arrayContainingInAnyOrder("index1-error", "index2-error")); Exception failure = response.getFailures().get(0).getException(); assertEquals(IllegalIndexShardStateException.class, failure.getClass()); - assertEquals("CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", failure.getMessage()); + assertEquals( + "CurrentState[CLOSED] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]", + failure.getMessage() + ); // the "indices" section should not include failed ones assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder("old_index", "new_index")); @@ -919,7 +922,7 @@ private void assertIndices(FieldCapabilitiesResponse response, String... indices static void closeShards(InternalTestCluster cluster, String... indices) throws IOException { final Set indicesToClose = Set.of(indices); - for (String node :cluster.getNodeNames()) { + for (String node : cluster.getNodeNames()) { final IndicesService indicesService = cluster.getInstance(IndicesService.class, node); for (IndexService indexService : indicesService) { if (indicesToClose.contains(indexService.getMetadata().getIndex().getName())) { From 6993e1c9d142d4b611adef6662bdf41d8f04acc4 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 9 Sep 2025 18:14:29 +0300 Subject: [PATCH 8/9] Add missing cluster feature checks --- .../test/inference/10_semantic_text_field_mapping.yml | 3 +++ .../test/inference/10_semantic_text_field_mapping_bwc.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml index 74a84a07a4c69..88f20d0c5fa6d 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml @@ -550,6 +550,9 @@ setup: --- "Field caps with semantic query does not fail": + - requires: + cluster_features: "semantic_query.filter_field_caps_fix" + reason: "fixed bug with semantic query filtering in field_caps (#116106)" # We need at least one document present to exercise can-match phase - do: index: diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml index 144185675388f..b184423836282 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml @@ -450,6 +450,9 @@ setup: --- "Field caps with semantic query does not fail": + - requires: + cluster_features: "semantic_query.filter_field_caps_fix" + reason: "fixed bug with semantic query filtering in field_caps (#116106)" # We need at least one document present to exercise can-match phase - do: index: From 933bf28fa6b8ec3c9ee2e23980d6de72e145dc2e Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 9 Sep 2025 18:19:41 +0300 Subject: [PATCH 9/9] Add changelog entry --- docs/changelog/134134.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/134134.yaml diff --git a/docs/changelog/134134.yaml b/docs/changelog/134134.yaml new file mode 100644 index 0000000000000..cbaff5bce55f6 --- /dev/null +++ b/docs/changelog/134134.yaml @@ -0,0 +1,6 @@ +pr: 134134 +summary: Prevent field caps from failing due to can match failure +area: Search +type: bug +issues: + - 116106