Refactor semantic text field to align with text field behaviour (elastic#119183)

Co-authored-by: Mike Pellegrini <[email protected]>

Author: jimczi

docs/reference/query-dsl/semantic-query.asciidoc

@@ -117,79 +117,3 @@ GET my-index/_search
 }
 ------------------------------------------------------------
 // TEST[skip: Requires inference endpoints]
-
-
-[discrete]
-[[advanced-search]]
-==== Advanced search on `semantic_text` fields
-
-The `semantic` query uses default settings for searching on `semantic_text` fields for ease of use.
-If you want to fine-tune a search on a `semantic_text` field, you need to know the task type used by the `inference_id` configured in `semantic_text`.
-You can find the task type using the <<get-inference-api>>, and check the `task_type` associated with the {infer} service.
-Depending on the `task_type`, use either the <<query-dsl-sparse-vector-query,`sparse_vector`>> or the <<query-dsl-knn-query,`knn`>> query for greater flexibility and customization.
-
-NOTE: While it is possible to use the `sparse_vector` query or the `knn` query
-on a `semantic_text` field, it is not supported to use the `semantic_query` on a
-`sparse_vector` or `dense_vector` field type.
-
-
-[discrete]
-[[search-sparse-inference]]
-===== Search with `sparse_embedding` inference
-
-When the {infer} endpoint uses a `sparse_embedding` model, you can use a <<query-dsl-sparse-vector-query,`sparse_vector` query>> on a <<semantic-text,`semantic_text`>> field in the following way:
-
-[source,console]
-------------------------------------------------------------
-GET test-index/_search
-{
-  "query": {
-    "nested": {
-      "path": "inference_field.inference.chunks",
-      "query": {
-        "sparse_vector": {
-          "field": "inference_field.inference.chunks.embeddings",
-          "inference_id": "my-inference-id",
-          "query": "mountain lake"
-        }
-      }
-    }
-  }
-}
-------------------------------------------------------------
-// TEST[skip: Requires inference endpoints]
-
-You can customize the `sparse_vector` query to include specific settings, like <<sparse-vector-query-with-pruning-config-and-rescore-example,pruning configuration>>.
-
-
-[discrete]
-[[search-text-inferece]]
-===== Search with `text_embedding` inference
-
-When the {infer} endpoint uses a `text_embedding` model, you can use a <<query-dsl-knn-query,`knn` query>> on a `semantic_text` field in the following way:
-
-[source,console]
-------------------------------------------------------------
-GET test-index/_search
-{
-  "query": {
-    "nested": {
-      "path": "inference_field.inference.chunks",
-      "query": {
-        "knn": {
-          "field": "inference_field.inference.chunks.embeddings",
-          "query_vector_builder": {
-            "text_embedding": {
-              "model_id": "my_inference_id",
-	      "model_text": "mountain lake"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-------------------------------------------------------------
-// TEST[skip: Requires inference endpoints]
-
-You can customize the `knn` query to include specific settings, like `num_candidates` and `k`.

docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc

@@ -157,89 +157,7 @@ GET semantic-embeddings/_search
 <2> The query text.
 
 As a result, you receive the top 10 documents that are closest in meaning to the
-query from the `semantic-embedding` index:
-
-[source,console-result]
-------------------------------------------------------------
-"hits": [
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Jy5065EBBFPLbFsdh_f9",
-    "_score": 21.487484,
-    "_source": {
-      "id": 8836652,
-      "content": {
-        "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  },
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Ji5065EBBFPLbFsdh_f9",
-    "_score": 18.211695,
-    "_source": {
-      "id": 8836651,
-      "content": {
-        "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  },
-  {
-    "_index": "semantic-embeddings",
-    "_id": "Wi5065EBBFPLbFsdh_b9",
-    "_score": 13.089405,
-    "_source": {
-      "id": 8800197,
-      "content": {
-        "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.",
-        "inference": {
-          "inference_id": "my-elser-endpoint",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.",
-              "embeddings": {
-                (...)
-              }
-            }
-          ]
-        }
-      }
-    }
-  }
-]
-------------------------------------------------------------
-// NOTCONSOLE
+query from the `semantic-embedding` index.
 
 [discrete]
 [[semantic-text-further-examples]]

muted-tests.yml

@@ -143,6 +143,9 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117349
+- class: org.elasticsearch.xpack.inference.InferenceRestIT
+  method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint}
+  issue: https://github.com/elastic/elasticsearch/issues/117349
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -48,9 +48,11 @@
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
@@ -326,7 +328,8 @@ static boolean executeBulkItemRequest(
         if (opType == DocWriteRequest.OpType.UPDATE) {
             final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
             try {
-                updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier);
+                var gFields = getStoredFieldsSpec(context.getPrimary());
+                updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier, gFields);
             } catch (Exception failure) {
                 // we may fail translating a update to index or delete operation
                 // we use index result to communicate failure while translating update request
@@ -401,6 +404,16 @@ static boolean executeBulkItemRequest(
         return true;
     }
 
+    private static String[] getStoredFieldsSpec(IndexShard indexShard) {
+        if (InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) {
+            if (indexShard.mapperService().mappingLookup().inferenceFields().size() > 0) {
+                // Retrieves the inference metadata field containing the inference results for all semantic fields defined in the mapping.
+                return new String[] { RoutingFieldMapper.NAME, InferenceMetadataFieldsMapper.NAME };
+            }
+        }
+        return new String[] { RoutingFieldMapper.NAME };
+    }
+
     private static boolean handleMappingUpdateRequired(
         BulkPrimaryExecutionContext context,
         MappingUpdatePerformer mappingUpdater,

server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java

@@ -44,6 +44,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.InferenceFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.IndexShard;
@@ -374,7 +375,7 @@ private static UpdateHelper.Result deleteInferenceResults(
         IndexMetadata indexMetadata,
         MappingLookup mappingLookup
     ) {
-        if (result.getResponseResult() != DocWriteResponse.Result.UPDATED) {
+        if (result.getResponseResult() != DocWriteResponse.Result.UPDATED || InferenceMetadataFieldsMapper.isEnabled(mappingLookup)) {
             return result;
         }
 
@@ -403,7 +404,7 @@ private static UpdateHelper.Result deleteInferenceResults(
             String inferenceFieldName = entry.getKey();
             Mapper mapper = mappingLookup.getMapper(inferenceFieldName);
 
-            if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) {
+            if (mapper instanceof InferenceFieldMapper) {
                 String[] sourceFields = entry.getValue().getSourceFields();
                 for (String sourceField : sourceFields) {
                     if (sourceField.equals(inferenceFieldName) == false
@@ -412,7 +413,7 @@ private static UpdateHelper.Result deleteInferenceResults(
                         // This has two important side effects:
                         // - The inference field value will remain parsable by its mapper
                         // - The inference results will be removed, forcing them to be re-generated downstream
-                        updatedSource.put(inferenceFieldName, inferenceFieldMapper.getOriginalValue(updatedSource));
+                        updatedSource.put(inferenceFieldName, getOriginalValueLegacy(inferenceFieldName, updatedSource));
                         updatedSourceModified = true;
                         break;
                     }
@@ -435,4 +436,24 @@ private static UpdateHelper.Result deleteInferenceResults(
 
         return returnedResult;
     }
+
+    /**
+     * Get the field's original value (i.e. the value the user specified) from the provided source.
+     *
+     * @param sourceAsMap The source as a map
+     * @return The field's original value, or {@code null} if none was provided
+     */
+    private static Object getOriginalValueLegacy(String fullPath, Map<String, Object> sourceAsMap) {
+        // TODO: Fix bug here when semantic text field is in an object
+        Object fieldValue = sourceAsMap.get(fullPath);
+        if (fieldValue == null) {
+            return null;
+        } else if (fieldValue instanceof Map<?, ?> == false) {
+            // Don't try to further validate the non-map value, that will be handled when the source is fully parsed
+            return fieldValue;
+        }
+
+        Map<String, Object> fieldValueMap = XContentMapValues.nodeMapValue(fieldValue, "Field [" + fullPath + "]");
+        return XContentMapValues.extractValue("text", fieldValueMap);
+    }
 }

server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java

@@ -60,7 +60,15 @@ public UpdateHelper(ScriptService scriptService) {
      * Prepares an update request by converting it into an index or delete request or an update response (no action).
      */
     public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) throws IOException {
-        final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm());
+        // TODO: Don't hard-code gFields
+        return prepare(request, indexShard, nowInMillis, new String[] { RoutingFieldMapper.NAME });
+    }
+
+    /**
+     * Prepares an update request by converting it into an index or delete request or an update response (no action).
+     */
+    public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis, String[] gFields) throws IOException {
+        final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm(), gFields);
         return prepare(indexShard, request, getResult, nowInMillis);
     }
 

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -190,6 +191,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
         SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
         IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
+        InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
 
         // validate that built-in similarities don't get redefined
         Setting.groupSetting("index.similarity.", (s) -> {

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -138,6 +138,8 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY = def(9_004_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java

@@ -440,7 +440,7 @@ private void readStoredFieldsDirectly(StoredFieldVisitor visitor) throws IOExcep
                 SourceFieldMapper mapper = mappingLookup.getMapping().getMetadataMapperByClass(SourceFieldMapper.class);
                 if (mapper != null) {
                     try {
-                        sourceBytes = mapper.applyFilters(sourceBytes, null);
+                        sourceBytes = mapper.applyFilters(mappingLookup, sourceBytes, null);
                     } catch (IOException e) {
                         throw new IOException("Failed to reapply filters after reading from translog", e);
                     }