Merge remote-tracking branch 'upstream/main' into fix-match-only-text

jordan-powers · jordan-powers · commit 4da3c7e23a54 · 2025-06-26T09:59:18.000-07:00
diff --git a/docs/changelog/129967.yaml b/docs/changelog/129967.yaml
@@ -0,0 +1,6 @@
+pr: 129967
+summary: Support returning default `index_options` for `semantic_text` fields when
+  `include_defaults` is true
+area: Search
+type: bug
+issues: []
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -524,9 +524,6 @@ tests:
 - class: org.elasticsearch.search.query.VectorIT
   method: testFilteredQueryStrategy
   issue: https://github.com/elastic/elasticsearch/issues/129517
-- class: org.elasticsearch.test.apmintegration.TracesApmIT
-  method: testApmIntegration
-  issue: https://github.com/elastic/elasticsearch/issues/129651
 - class: org.elasticsearch.snapshots.SnapshotShutdownIT
   method: testSnapshotShutdownProgressTracker
   issue: https://github.com/elastic/elasticsearch/issues/129752
@@ -555,8 +552,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
   method: testCohereEmbeddings
   issue: https://github.com/elastic/elasticsearch/issues/130010
-- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
-  issue: https://github.com/elastic/elasticsearch/issues/128224
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/130067
diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/RecordingApmServer.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/RecordingApmServer.java
@@ -15,7 +15,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.xcontent.spi.XContentProvider;
 import org.junit.rules.ExternalResource;
 
 import java.io.BufferedReader;
@@ -25,7 +24,6 @@
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -35,14 +33,12 @@
 public class RecordingApmServer extends ExternalResource {
     private static final Logger logger = LogManager.getLogger(RecordingApmServer.class);
 
-    private static final XContentProvider.FormatProvider XCONTENT = XContentProvider.provider().getJsonXContent();
-
     final ArrayBlockingQueue<String> received = new ArrayBlockingQueue<>(1000);
 
     private static HttpServer server;
     private final Thread messageConsumerThread = consumerThread();
     private volatile Consumer<String> consumer;
-    private volatile boolean consumerRunning = true;
+    private volatile boolean running = true;
 
     @Override
     protected void before() throws Throwable {
@@ -56,7 +52,7 @@ protected void before() throws Throwable {
 
     private Thread consumerThread() {
         return new Thread(() -> {
-            while (consumerRunning) {
+            while (running) {
                 if (consumer != null) {
                     try {
                         String msg = received.poll(1L, TimeUnit.SECONDS);
@@ -74,28 +70,38 @@ private Thread consumerThread() {
 
     @Override
     protected void after() {
+        running = false;
         server.stop(1);
-        consumerRunning = false;
+        consumer = null;
     }
 
     private void handle(HttpExchange exchange) throws IOException {
         try (exchange) {
-            try {
-                try (InputStream requestBody = exchange.getRequestBody()) {
-                    if (requestBody != null) {
-                        var read = readJsonMessages(requestBody);
-                        received.addAll(read);
+            if (running) {
+                try {
+                    try (InputStream requestBody = exchange.getRequestBody()) {
+                        if (requestBody != null) {
+                            var read = readJsonMessages(requestBody);
+                            received.addAll(read);
+                        }
                     }
-                }
 
-            } catch (RuntimeException e) {
-                logger.warn("failed to parse request", e);
+                } catch (Throwable t) {
+                    // The lifetime of HttpServer makes message handling "brittle": we need to start handling and recording received
+                    // messages before the test starts running. We should also stop handling them before the test ends (and the test
+                    // cluster is torn down), or we may run into IOException as the communication channel is interrupted.
+                    // Coordinating the lifecycle of the mock HttpServer and of the test ES cluster is difficult and error-prone, so
+                    // we just handle Throwable and don't care (log, but don't care): if we have an error in communicating to/from
+                    // the mock server while the test is running, the test would fail anyway as the expected messages will not arrive, and
+                    // if we have an error outside the test scope (before or after) that is OK.
+                    logger.warn("failed to parse request", t);
+                }
             }
             exchange.sendResponseHeaders(201, 0);
         }
     }
 
-    private List<String> readJsonMessages(InputStream input) throws IOException {
+    private List<String> readJsonMessages(InputStream input) {
         // parse NDJSON
         return new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)).lines().toList();
     }
@@ -104,14 +110,7 @@ public int getPort() {
         return server.getAddress().getPort();
     }
 
-    public List<String> getMessages() {
-        List<String> list = new ArrayList<>(received.size());
-        received.drainTo(list);
-        return list;
-    }
-
     public void addMessageConsumer(Consumer<String> messageConsumer) {
         this.consumer = messageConsumer;
     }
-
 }
diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/TracesApmIT.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/TracesApmIT.java
@@ -91,7 +91,8 @@ public void testApmIntegration() throws Exception {
 
         client().performRequest(nodeStatsRequest);
 
-        finished.await(30, TimeUnit.SECONDS);
+        var completed = finished.await(30, TimeUnit.SECONDS);
+        assertTrue("Timeout when waiting for assertions to complete", completed);
         assertThat(assertions, equalTo(Collections.emptySet()));
     }
 
@@ -143,5 +144,4 @@ private Map<String, Object> parseMap(String message) {
             return Collections.emptyMap();
         }
     }
-
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-colors.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-colors.json
@@ -13,7 +13,9 @@
       "type": "dense_vector",
       "similarity": "l2_norm",
       "index_options": {
-        "type": "hnsw"
+        "type": "hnsw",
+        "m": 16,
+        "ef_construction": 100
       }
     }
   }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-dense_vector.json
@@ -5,7 +5,12 @@
     },
     "vector": {
       "type": "dense_vector",
-      "similarity": "l2_norm"
+      "similarity": "l2_norm",
+      "index_options": {
+        "type": "hnsw",
+        "m": 16,
+        "ef_construction": 100
+      }
     }
   }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
@@ -17,6 +17,7 @@
 
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
@@ -66,7 +67,8 @@ public Set<NodeFeature> getTestFeatures() {
             SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER,
             SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS,
             SEMANTIC_TEXT_INDEX_OPTIONS,
-            COHERE_V2_API
+            COHERE_V2_API,
+            SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -69,6 +69,7 @@
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.MinimalServiceSettings;
 import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
@@ -139,6 +140,9 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
         "semantic_text.exclude_sub_fields_from_field_caps"
     );
     public static final NodeFeature SEMANTIC_TEXT_INDEX_OPTIONS = new NodeFeature("semantic_text.index_options");
+    public static final NodeFeature SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS = new NodeFeature(
+        "semantic_text.index_options_with_defaults"
+    );
 
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
@@ -166,19 +170,9 @@ public static BiConsumer<String, MappingParserContext> validateParserContext(Str
     public static class Builder extends FieldMapper.Builder {
         private final ModelRegistry modelRegistry;
         private final boolean useLegacyFormat;
+        private final IndexVersion indexVersionCreated;
 
-        private final Parameter<String> inferenceId = Parameter.stringParam(
-            INFERENCE_ID_FIELD,
-            false,
-            mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
-            DEFAULT_ELSER_2_INFERENCE_ID
-        ).addValidator(v -> {
-            if (Strings.isEmpty(v)) {
-                throw new IllegalArgumentException(
-                    "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
-                );
-            }
-        }).alwaysSerialize();
+        private final Parameter<String> inferenceId;
 
         private final Parameter<String> searchInferenceId = Parameter.stringParam(
             SEARCH_INFERENCE_ID_FIELD,
@@ -193,25 +187,9 @@ public static class Builder extends FieldMapper.Builder {
             }
         });
 
-        private final Parameter<MinimalServiceSettings> modelSettings = new Parameter<>(
-            MODEL_SETTINGS_FIELD,
-            true,
-            () -> null,
-            (n, c, o) -> SemanticTextField.parseModelSettingsFromMap(o),
-            mapper -> ((SemanticTextFieldType) mapper.fieldType()).modelSettings,
-            XContentBuilder::field,
-            Objects::toString
-        ).acceptsNull().setMergeValidator(SemanticTextFieldMapper::canMergeModelSettings);
+        private final Parameter<MinimalServiceSettings> modelSettings;
 
-        private final Parameter<SemanticTextIndexOptions> indexOptions = new Parameter<>(
-            INDEX_OPTIONS_FIELD,
-            true,
-            () -> null,
-            (n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
-            mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
-            XContentBuilder::field,
-            Objects::toString
-        ).acceptsNull();
+        private final Parameter<SemanticTextIndexOptions> indexOptions;
 
         @SuppressWarnings("unchecked")
         private final Parameter<ChunkingSettings> chunkingSettings = new Parameter<>(
@@ -248,6 +226,50 @@ public Builder(
             super(name);
             this.modelRegistry = modelRegistry;
             this.useLegacyFormat = InferenceMetadataFieldsMapper.isEnabled(indexSettings.getSettings()) == false;
+            this.indexVersionCreated = indexSettings.getIndexVersionCreated();
+
+            this.inferenceId = Parameter.stringParam(
+                INFERENCE_ID_FIELD,
+                false,
+                mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
+                DEFAULT_ELSER_2_INFERENCE_ID
+            ).addValidator(v -> {
+                if (Strings.isEmpty(v)) {
+                    throw new IllegalArgumentException(
+                        "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
+                    );
+                }
+            }).alwaysSerialize();
+
+            this.modelSettings = new Parameter<>(
+                MODEL_SETTINGS_FIELD,
+                true,
+                () -> null,
+                (n, c, o) -> SemanticTextField.parseModelSettingsFromMap(o),
+                mapper -> ((SemanticTextFieldType) mapper.fieldType()).modelSettings,
+                XContentBuilder::field,
+                Objects::toString
+            ).acceptsNull().setMergeValidator(SemanticTextFieldMapper::canMergeModelSettings);
+
+            this.indexOptions = new Parameter<>(
+                INDEX_OPTIONS_FIELD,
+                true,
+                () -> null,
+                (n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
+                mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
+                (b, n, v) -> {
+                    if (v == null) {
+                        MinimalServiceSettings resolvedModelSettings = modelSettings.get() != null
+                            ? modelSettings.get()
+                            : modelRegistry.getMinimalServiceSettings(inferenceId.get());
+                        b.field(INDEX_OPTIONS_FIELD, defaultIndexOptions(indexVersionCreated, resolvedModelSettings));
+                    } else {
+                        b.field(INDEX_OPTIONS_FIELD, v);
+                    }
+                },
+                Objects::toString
+            ).acceptsNull();
+
             this.inferenceFieldBuilder = c -> {
                 // Resolve the model setting from the registry if it has not been set yet.
                 var resolvedModelSettings = modelSettings.get() != null ? modelSettings.get() : getResolvedModelSettings(c, false);
@@ -365,8 +387,11 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) {
                 validateServiceSettings(modelSettings.get(), resolvedModelSettings);
             }
 
-            if (context.getMergeReason() != MapperService.MergeReason.MAPPING_RECOVERY && indexOptions.get() != null) {
-                validateIndexOptions(indexOptions.get(), inferenceId.getValue(), resolvedModelSettings);
+            // If index_options are specified by the user, we will validate them against the model settings to ensure compatibility.
+            // We do not serialize or otherwise store model settings at this time, this happens when the underlying vector field is created.
+            SemanticTextIndexOptions builderIndexOptions = indexOptions.get();
+            if (context.getMergeReason() != MapperService.MergeReason.MAPPING_RECOVERY && builderIndexOptions != null) {
+                validateIndexOptions(builderIndexOptions, inferenceId.getValue(), resolvedModelSettings);
             }
 
             final String fullName = context.buildFullName(leafName());
@@ -1166,6 +1191,9 @@ private static Mapper.Builder createEmbeddingsField(
                 }
                 denseVectorMapperBuilder.dimensions(modelSettings.dimensions());
                 denseVectorMapperBuilder.elementType(modelSettings.elementType());
+                // Here is where we persist index_options. If they are specified by the user, we will use those index_options,
+                // otherwise we will determine if we can set default index options. If we can't, we won't persist any index_options
+                // and the field will use the defaults for the dense_vector field.
                 if (indexOptions != null) {
                     DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions =
                         (DenseVectorFieldMapper.DenseVectorIndexOptions) indexOptions.indexOptions();
@@ -1208,7 +1236,6 @@ static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVectorIndexOpt
         // As embedding models for text perform better with BBQ, we aggressively default semantic_text fields to use optimized index
         // options
         if (indexVersionDefaultsToBbqHnsw(indexVersionCreated)) {
-
             DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswIndexOptions = defaultBbqHnswDenseVectorIndexOptions();
             return defaultBbqHnswIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false)
                 ? defaultBbqHnswIndexOptions
@@ -1230,11 +1257,24 @@ static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorI
         return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector);
     }
 
-    static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() {
-        return new SemanticTextIndexOptions(
-            SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR,
-            defaultBbqHnswDenseVectorIndexOptions()
-        );
+    static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings) {
+
+        if (modelSettings == null) {
+            return null;
+        }
+
+        SemanticTextIndexOptions defaultIndexOptions = null;
+        if (modelSettings.taskType() == TaskType.TEXT_EMBEDDING) {
+            DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions = defaultDenseVectorIndexOptions(
+                indexVersionCreated,
+                modelSettings
+            );
+            defaultIndexOptions = denseVectorIndexOptions == null
+                ? null
+                : new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseVectorIndexOptions);
+        }
+
+        return defaultIndexOptions;
     }
 
     private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml

Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,8 @@ public void testApmIntegration() throws Exception {`
`91`	`91`
`92`	`92`	`client().performRequest(nodeStatsRequest);`
`93`	`93`
`94`		`- finished.await(30, TimeUnit.SECONDS);`
	`94`	`+ var completed = finished.await(30, TimeUnit.SECONDS);`
	`95`	`+ assertTrue("Timeout when waiting for assertions to complete", completed);`
`95`	`96`	`assertThat(assertions, equalTo(Collections.emptySet()));`
`96`	`97`	`}`
`97`	`98`
`@@ -143,5 +144,4 @@ private Map<String, Object> parseMap(String message) {`
`143`	`144`	`return Collections.emptyMap();`
`144`	`145`	`}`
`145`	`146`	`}`
`146`		`-`
`147`	`147`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,9 @@`
`13`	`13`	`"type": "dense_vector",`
`14`	`14`	`"similarity": "l2_norm",`
`15`	`15`	`"index_options": {`
`16`		`- "type": "hnsw"`
	`16`	`+ "type": "hnsw",`
	`17`	`+ "m": 16,`
	`18`	`+ "ef_construction": 100`
`17`	`19`	`}`
`18`	`20`	`}`
`19`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,12 @@`
`5`	`5`	`},`
`6`	`6`	`"vector": {`
`7`	`7`	`"type": "dense_vector",`
`8`		`- "similarity": "l2_norm"`
	`8`	`+ "similarity": "l2_norm",`
	`9`	`+ "index_options": {`
	`10`	`+ "type": "hnsw",`
	`11`	`+ "m": 16,`
	`12`	`+ "ef_construction": 100`
	`13`	`+ }`
`9`	`14`	`}`
`10`	`15`	`}`
`11`	`16`	`}`