fix(semantic_text): index underlying dense_vector field in older indices

Author: mromaios

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -406,6 +406,11 @@ protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { elementType, dims, indexed, similarity, indexOptions, meta };
         }
 
+        public Builder indexed(boolean indexed) {
+            this.indexed.setValue(indexed);
+            return this;
+        }
+
         public Builder similarity(VectorSimilarity vectorSimilarity) {
             similarity.setValue(vectorSimilarity);
             return this;

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -101,8 +102,10 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
 import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ;
 import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X;
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.INDEXED_BY_DEFAULT_INDEX_VERSION;
 import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING;
 import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING;
 import static org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
@@ -127,6 +130,11 @@
  */
 public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
     private static final Logger logger = LogManager.getLogger(SemanticTextFieldMapper.class);
+    // TODO: rewrite the warning and error messages below, just placeholders for now
+    static final String WARNING_MESSAGE_8X = "The [dense_vector] field type created by `semantic_text` uses default index settings which may not be optimal. " +
+        "Consider creating a new index ....";
+    private static final String ERROR_MESSAGE_UNSUPPORTED_SPARSE_VECTOR = "The [sparse_vector] field type created by `semantic_text` is not supported on indices created on versions 8.0 to 8.10." +
+        "Try using a `dense_vector` model or upgrade your index to 8.11 or later. ...";
     public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix");
     public static final NodeFeature SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX = new NodeFeature("semantic_text.single_field_update_fix");
     public static final NodeFeature SEMANTIC_TEXT_DELETE_FIX = new NodeFeature("semantic_text.delete_fix");
@@ -1276,6 +1284,10 @@ private static void configureSparseVectorMapperBuilder(
         SparseVectorFieldMapper.Builder sparseVectorMapperBuilder,
         SemanticTextIndexOptions indexOptions
     ) {
+        if (indexVersionCreated.before(NEW_SPARSE_VECTOR)) {
+            throw new IllegalArgumentException(ERROR_MESSAGE_UNSUPPORTED_SPARSE_VECTOR);
+        }
+
         if (indexOptions != null) {
             SparseVectorFieldMapper.SparseVectorIndexOptions sparseVectorIndexOptions =
                 (SparseVectorFieldMapper.SparseVectorIndexOptions) indexOptions.indexOptions();
@@ -1296,6 +1308,13 @@ private static void configureDenseVectorMapperBuilder(
         MinimalServiceSettings modelSettings,
         SemanticTextIndexOptions indexOptions
     ) {
+        if (indexVersionCreated.onOrAfter(IndexVersions.V_8_0_0)){
+            if (indexVersionCreated.before(INDEXED_BY_DEFAULT_INDEX_VERSION)) {
+                deprecationLogger.warn(DeprecationCategory.MAPPINGS, "semantic_text", WARNING_MESSAGE_8X);
+            }
+            denseVectorMapperBuilder.indexed(true);
+        }
+
         SimilarityMeasure similarity = modelSettings.similarity();
         if (similarity != null) {
             switch (similarity) {