Use ELSER By Default For Semantic Text (#113563) (#114346)

Co-authored-by: David Kyle <[email protected]>
(cherry picked from commit 0cc0544)

# Conflicts:
#	x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

Co-authored-by: Elastic Machine <[email protected]>

Author: Mikep86

docs/changelog/113563.yaml

@@ -0,0 +1,5 @@
+pr: 113563
+summary: Use ELSER By Default For Semantic Text
+area: Mapping
+type: enhancement
+issues: []

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
 
+import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -22,12 +23,15 @@ public class InferenceFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(
-            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED,
-            RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED,
-            SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID,
-            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED
-        );
+        var features = new HashSet<NodeFeature>();
+        features.add(TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED);
+        features.add(RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED);
+        features.add(SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID);
+        features.add(TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED);
+        if (DefaultElserFeatureFlag.isEnabled()) {
+            features.add(SemanticTextFieldMapper.SEMANTIC_TEXT_DEFAULT_ELSER_2);
+        }
+        return Set.copyOf(features);
     }
 
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java

@@ -58,6 +58,7 @@ public record SemanticTextField(String fieldName, List<String> originalValues, I
     static final String TEXT_FIELD = "text";
     static final String INFERENCE_FIELD = "inference";
     static final String INFERENCE_ID_FIELD = "inference_id";
+    static final String SEARCH_INFERENCE_ID_FIELD = "search_inference_id";
     static final String CHUNKS_FIELD = "chunks";
     static final String CHUNKED_EMBEDDINGS_FIELD = "embeddings";
     static final String CHUNKED_TEXT_FIELD = "text";

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -54,6 +54,7 @@
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
+import org.elasticsearch.xpack.inference.DefaultElserFeatureFlag;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -71,18 +72,23 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName;
+import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.DEFAULT_ELSER_ID;
 
 /**
  * A {@link FieldMapper} for semantic text fields.
  */
 public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
     public static final NodeFeature SEMANTIC_TEXT_SEARCH_INFERENCE_ID = new NodeFeature("semantic_text.search_inference_id");
+    public static final NodeFeature SEMANTIC_TEXT_DEFAULT_ELSER_2 = new NodeFeature("semantic_text.default_elser_2");
 
     public static final String CONTENT_TYPE = "semantic_text";
+    public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
 
     private final IndexSettings indexSettings;
 
@@ -96,25 +102,37 @@ public static class Builder extends FieldMapper.Builder {
         private final IndexSettings indexSettings;
 
         private final Parameter<String> inferenceId = Parameter.stringParam(
-            "inference_id",
+            INFERENCE_ID_FIELD,
             false,
             mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
-            null
+            DefaultElserFeatureFlag.isEnabled() ? DEFAULT_ELSER_2_INFERENCE_ID : null
         ).addValidator(v -> {
             if (Strings.isEmpty(v)) {
-                throw new IllegalArgumentException("field [inference_id] must be specified");
+                // If the default ELSER feature flag is enabled, the only way we get here is if the user explicitly sets the param to an
+                // empty value. However, if the feature flag is disabled, we can get here if the user didn't set the param.
+                // Adjust the error message appropriately.
+                String message = DefaultElserFeatureFlag.isEnabled()
+                    ? "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
+                    : "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must be specified";
+                throw new IllegalArgumentException(message);
             }
         });
 
         private final Parameter<String> searchInferenceId = Parameter.stringParam(
-            "search_inference_id",
+            SEARCH_INFERENCE_ID_FIELD,
             true,
             mapper -> ((SemanticTextFieldType) mapper.fieldType()).searchInferenceId,
             null
-        ).acceptsNull();
+        ).acceptsNull().addValidator(v -> {
+            if (v != null && Strings.isEmpty(v)) {
+                throw new IllegalArgumentException(
+                    "[" + SEARCH_INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
+                );
+            }
+        });
 
         private final Parameter<SemanticTextField.ModelSettings> modelSettings = new Parameter<>(
-            "model_settings",
+            MODEL_SETTINGS_FIELD,
             true,
             () -> null,
             (n, c, o) -> SemanticTextField.parseModelSettingsFromMap(o),
@@ -204,6 +222,7 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) {
             }
             var childContext = context.createChildContext(leafName(), ObjectMapper.Dynamic.FALSE);
             final ObjectMapper inferenceField = inferenceFieldBuilder.apply(childContext);
+
             return new SemanticTextFieldMapper(
                 leafName(),
                 new SemanticTextFieldType(