Default semantic_text fields to ELSER on EIS when available (#134708)

* Defaulting EIS on ELSER

* Extended testing

* [CI] Auto commit changes from spotless

* Edited to include the default

* Cleaned up the mapper implementation

* COmpile issue

* [CI] Auto commit changes from spotless

* Added tests

* [CI] Auto commit changes from spotless

* Refactored the variable names

* Cleanup done

* Removed unnecessary files

* Unit tests and mock is working

* [CI] Auto commit changes from spotless

* Fix test

* yaml addition failure

* [CI] Auto commit changes from spotless

* Resolved the import issue or duplication of variables in mock

* Resolved PR comments

* Restored error

* Update docs/changelog/134708.yaml

* Integration test

* [CI] Auto commit changes from spotless

* Resolved all PR comments

* [CI] Auto commit changes from spotless

* Cleaned up the redudant reference of TestInferencePlugin

* Included both before and before test

* [CI] Auto commit changes from spotless

* Made changes to accomodate the old constant version string

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: mridula-s109

docs/changelog/134708.yaml

@@ -0,0 +1,5 @@
+pr: 134708
+summary: Default `semantic_text` fields to ELSER on EIS when available
+area: Mapping
+type: enhancement
+issues: []

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/SemanticTextEISDefaultIT.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V.
+ * Licensed under the Elastic License 2.0; you may not use this file except
+ * in compliance with the Elastic License 2.0.
+ */
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_EIS_ELSER_INFERENCE_ID;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * End-to-end test that verifies semantic_text fields automatically default to ELSER on EIS
+ * when available and no inference_id is explicitly provided.
+ */
+public class SemanticTextEISDefaultIT extends BaseMockEISAuthServerTest {
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        // Ensure the mock EIS server has an authorized response ready before each test
+        mockEISServer.enqueueAuthorizeAllModelsResponse();
+    }
+
+    /**
+     * This is done before the class because I've run into issues where another class that extends {@link BaseMockEISAuthServerTest}
+     * results in an authorization response not being queued up for the new Elasticsearch Node in time. When the node starts up, it
+     * retrieves authorization. If the request isn't queued up when that happens the tests will fail. From my testing locally it seems
+     * like the base class's static functionality to queue a response is only done once and not for each subclass.
+     *
+     * My understanding is that the @Before will be run after the node starts up and wouldn't be sufficient to handle
+     * this scenario. That is why this needs to be @BeforeClass.
+     */
+    @BeforeClass
+    public static void init() {
+        // Ensure the mock EIS server has an authorized response ready
+        mockEISServer.enqueueAuthorizeAllModelsResponse();
+    }
+
+    public void testDefaultInferenceIdForSemanticText() throws IOException {
+        String indexName = "semantic-index";
+        String mapping = """
+            {
+              "properties": {
+                "semantic_text_field": {
+                  "type": "semantic_text"
+                }
+              }
+            }
+            """;
+        Settings settings = Settings.builder().build();
+        createIndex(indexName, settings, mapping);
+
+        Map<String, Object> mappingAsMap = getIndexMappingAsMap(indexName);
+        String populatedInferenceId = (String) XContentMapValues.extractValue("properties.semantic_text_field.inference_id", mappingAsMap);
+
+        assertThat(
+            "semantic_text field should default to ELSER on EIS when available",
+            populatedInferenceId,
+            equalTo(DEFAULT_EIS_ELSER_INFERENCE_ID)
+        );
+    }
+
+}

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -121,6 +121,7 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOffsetsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName;
+import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.DEFAULT_ELSER_ENDPOINT_ID_V2;
 import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.DEFAULT_ELSER_ID;
 
 /**
@@ -153,8 +154,13 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final NodeFeature SEMANTIC_TEXT_UPDATABLE_INFERENCE_ID = new NodeFeature("semantic_text.updatable_inference_id");
 
     public static final String CONTENT_TYPE = "semantic_text";
-    public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
-
+    public static final String DEFAULT_FALLBACK_ELSER_INFERENCE_ID = DEFAULT_ELSER_ID;
+    public static final String DEFAULT_EIS_ELSER_INFERENCE_ID = DEFAULT_ELSER_ENDPOINT_ID_V2;
+    /**
+     * @deprecated Replaced by {@link #DEFAULT_EIS_ELSER_INFERENCE_ID}.
+     */
+    @Deprecated(since = "9.3.0", forRemoval = false)
+    public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_EIS_ELSER_INFERENCE_ID;
     public static final String UNSUPPORTED_INDEX_MESSAGE = "["
         + CONTENT_TYPE
         + "] is available on indices created with 8.11 or higher. Please create a new index to use ["
@@ -163,6 +169,18 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
 
     public static final float DEFAULT_RESCORE_OVERSAMPLE = 3.0f;
 
+    /**
+     * Determines the preferred ELSER inference ID based on EIS availability.
+     * Returns .elser-2-elastic (EIS) when available, otherwise falls back to .elser-2-elasticsearch (ML nodes).
+     * This enables automatic selection of EIS for better performance while maintaining compatibility with on-prem deployments.
+     */
+    private static String getPreferredElserInferenceId(ModelRegistry modelRegistry) {
+        if (modelRegistry != null && modelRegistry.containsDefaultConfigId(DEFAULT_EIS_ELSER_INFERENCE_ID)) {
+            return DEFAULT_EIS_ELSER_INFERENCE_ID;
+        }
+        return DEFAULT_FALLBACK_ELSER_INFERENCE_ID;
+    }
+
     static final String INDEX_OPTIONS_FIELD = "index_options";
 
     public static final TypeParser parser(Supplier<ModelRegistry> modelRegistry) {
@@ -246,7 +264,7 @@ public Builder(
                 INFERENCE_ID_FIELD,
                 true,
                 mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
-                DEFAULT_ELSER_2_INFERENCE_ID
+                getPreferredElserInferenceId(modelRegistry)
             ).addValidator(v -> {
                 if (Strings.isEmpty(v)) {
                     throw new IllegalArgumentException(
@@ -926,7 +944,8 @@ public Query termQuery(Object value, SearchExecutionContext context) {
 
         @Override
         public Query existsQuery(SearchExecutionContext context) {
-            if (getEmbeddingsField() == null) {
+            // If this field has never seen inference results (no model settings), there are no values yet
+            if (modelSettings == null) {
                 return new MatchNoDocsQuery();
             }
 

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java

@@ -114,7 +114,7 @@ public class ElasticInferenceService extends SenderService {
 
     // elser-2
     static final String DEFAULT_ELSER_2_MODEL_ID = "elser_model_2";
-    static final String DEFAULT_ELSER_ENDPOINT_ID_V2 = defaultEndpointId("elser-2");
+    public static final String DEFAULT_ELSER_ENDPOINT_ID_V2 = defaultEndpointId("elser-2");
 
     // multilingual-text-embed
     static final String DEFAULT_MULTILINGUAL_EMBED_MODEL_ID = "jina-embeddings-v3";

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java

@@ -26,6 +26,7 @@
 import org.apache.lucene.search.join.QueryBitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.CheckedBiConsumer;
@@ -63,6 +64,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
 import org.elasticsearch.inference.ChunkingSettings;
+import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.MinimalServiceSettings;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ServiceSettings;
@@ -84,6 +86,7 @@
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.model.TestModel;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService;
 import org.junit.After;
 import org.junit.AssumptionViolatedException;
 import org.junit.Before;
@@ -111,7 +114,9 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_EIS_ELSER_INFERENCE_ID;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_FALLBACK_ELSER_INFERENCE_ID;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.INDEX_OPTIONS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.UNSUPPORTED_INDEX_MESSAGE;
@@ -124,6 +129,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
@@ -139,7 +145,7 @@ public SemanticTextFieldMapperTests(boolean useLegacyFormat) {
     ModelRegistry globalModelRegistry;
 
     @Before
-    private void startThreadPool() {
+    private void initializeTestEnvironment() {
         threadPool = createThreadPool();
         var clusterService = ClusterServiceUtils.createClusterService(threadPool);
         var modelRegistry = new ModelRegistry(clusterService, new NoOpClient(threadPool));
@@ -150,6 +156,7 @@ public boolean localNodeMaster() {
                 return false;
             }
         });
+        registerDefaultEisEndpoint();
     }
 
     @After
@@ -172,6 +179,16 @@ protected Supplier<ModelRegistry> getModelRegistry() {
         }, new XPackClientPlugin());
     }
 
+    private void registerDefaultEisEndpoint() {
+        globalModelRegistry.putDefaultIdIfAbsent(
+            new InferenceService.DefaultConfigId(
+                DEFAULT_EIS_ELSER_INFERENCE_ID,
+                MinimalServiceSettings.sparseEmbedding(ElasticInferenceService.NAME),
+                mock(InferenceService.class)
+            )
+        );
+    }
+
     private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException {
         IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat);
         return createMapperService(mappings, useLegacyFormat, indexVersion, indexVersion);
@@ -226,7 +243,7 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
     @Override
     protected void metaMapping(XContentBuilder b) throws IOException {
         super.metaMapping(b);
-        b.field(INFERENCE_ID_FIELD, DEFAULT_ELSER_2_INFERENCE_ID);
+        b.field(INFERENCE_ID_FIELD, DEFAULT_EIS_ELSER_INFERENCE_ID);
     }
 
     @Override
@@ -293,7 +310,7 @@ public void testDefaults() throws Exception {
         DocumentMapper mapper = mapperService.documentMapper();
         assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString());
         assertSemanticTextField(mapperService, fieldName, false, null, null);
-        assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, DEFAULT_ELSER_2_INFERENCE_ID);
+        assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_ELSER_INFERENCE_ID, DEFAULT_EIS_ELSER_INFERENCE_ID);
 
         ParsedDocument doc1 = mapper.parse(source(this::writeField));
         List<IndexableField> fields = doc1.rootDoc().getFields("field");
@@ -302,6 +319,30 @@ public void testDefaults() throws Exception {
         assertTrue(fields.isEmpty());
     }
 
+    public void testDefaultInferenceIdUsesEisWhenAvailable() throws Exception {
+        final String fieldName = "field";
+        final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping);
+
+        MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
+        assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_ELSER_INFERENCE_ID, DEFAULT_EIS_ELSER_INFERENCE_ID);
+    }
+
+    public void testDefaultInferenceIdFallsBackWhenEisUnavailable() throws Exception {
+        final String fieldName = "field";
+        final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping);
+
+        removeDefaultEisEndpoint();
+
+        MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
+        assertInferenceEndpoints(mapperService, fieldName, DEFAULT_FALLBACK_ELSER_INFERENCE_ID, DEFAULT_FALLBACK_ELSER_INFERENCE_ID);
+    }
+
+    private void removeDefaultEisEndpoint() {
+        PlainActionFuture<Boolean> removalFuture = new PlainActionFuture<>();
+        globalModelRegistry.removeDefaultConfigs(Set.of(DEFAULT_EIS_ELSER_INFERENCE_ID), removalFuture);
+        assertTrue("Failed to remove default EIS endpoint", removalFuture.actionGet(TEST_REQUEST_TIMEOUT));
+    }
+
     @Override
     public void testFieldHasValue() {
         MappedFieldType fieldType = getMappedFieldType();
@@ -332,12 +373,12 @@ public void testSetInferenceEndpoints() throws IOException {
             );
             final XContentBuilder expectedMapping = fieldMapping(
                 b -> b.field("type", "semantic_text")
-                    .field(INFERENCE_ID_FIELD, DEFAULT_ELSER_2_INFERENCE_ID)
+                    .field(INFERENCE_ID_FIELD, DEFAULT_EIS_ELSER_INFERENCE_ID)
                     .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId)
             );
             final MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat);
             assertSemanticTextField(mapperService, fieldName, false, null, null);
-            assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, searchInferenceId);
+            assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_ELSER_INFERENCE_ID, searchInferenceId);
             assertSerialization.accept(expectedMapping, mapperService);
         }
         {