Merge branch 'main' into lucene_snapshot_10_1

Author: ChrisHegarty

.buildkite/pipelines/pull-request/part-4-entitlements.yml

@@ -7,5 +7,5 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-fips.yml

@@ -7,5 +7,5 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4.yml

@@ -7,5 +7,5 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk

docs/changelog/117504.yaml

@@ -0,0 +1,6 @@
+pr: 117504
+summary: Fix NPE caused by race condition in async search when minimise round trips
+  is true
+area: Search
+type: bug
+issues: []

docs/changelog/119003.yaml

@@ -0,0 +1,5 @@
+pr: 119003
+summary: Add a `replicate_for` option to the ILM `searchable_snapshot` action
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/120527.yaml

@@ -0,0 +1,6 @@
+pr: 120527
+summary: Implement a `MetricsAware` interface
+area: ES|QL
+type: enhancement
+issues:
+ - 115992

docs/changelog/120813.yaml

@@ -0,0 +1,17 @@
+pr: 120813
+summary: Change Semantic Text To Act Like A Normal Text Field
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Change Semantic Text To Act Like A Normal Text Field
+  area: Search
+  details:
+    The previous semantic_text format used a complex subfield structure in _source to store the embeddings.
+    This complicated interactions/integrations with semantic_text fields and _source in general.
+    This new semantic_text format treats it as a normal text field, where the field's value in _source is the value assigned by the user.
+  impact:
+    Users who parsed the subfield structure of the previous semantic_text format in _source will need to update their parsing logic.
+    The new format does not directly expose the chunks and embeddings generated from the input text.
+    The new format will be applied to all new indices, any existing indices will continue to use the previous format.
+  notable: true

docs/changelog/120833.yaml

@@ -0,0 +1,5 @@
+pr: 120833
+summary: Optimize `IngestCtxMap` construction
+area: Ingest Node
+type: enhancement
+issues: []

docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc

@@ -55,6 +55,13 @@ snapshot retention runs off the index lifecycle management (ILM) policies and is
 (Required, string)
 <<snapshots-register-repository,Repository>> used to store the snapshot.
 
+`replicate_for`::
+(Optional, TimeValue)
+By default, searchable snapshot indices are mounted without replicas. Using this will
+result in a searchable snapshot index being mounted with a single replica for the time period
+specified, after which the replica will be removed. This option is only permitted on the
+first searchable snapshot action of a policy.
+
 `force_merge_index`::
 (Optional, Boolean)
 Force merges the managed index to one segment.
@@ -109,3 +116,44 @@ PUT _ilm/policy/my_policy
   }
 }
 --------------------------------------------------
+
+[[ilm-searchable-snapshot-replicate-for-ex]]
+===== Mount a searchable snapshot with replicas for fourteen days
+
+This policy mounts a searchable snapshot in the hot phase with a
+single replica and maintains that replica for fourteen days. After
+that time has elapsed, the searchable snapshot index will remain (with
+no replicas) for another fourteen days, at which point it will proceed
+into the delete phase and will be deleted.
+
+[source,console]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover" : {
+            "max_primary_shard_size": "50gb"
+          },
+          "searchable_snapshot" : {
+            "snapshot_repository" : "backing_repo",
+            "replicate_for": "14d"
+          }
+        }
+      },
+      "delete": {
+        "min_age": "28d",
+        "actions": {
+          "delete" : { }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+[NOTE]
+If the `replicate_for` option is specified, its value must be
+less than the minimum age of the next phase in the policy.

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java

@@ -527,7 +527,7 @@ public void testAutoConvertNotString() throws Exception {
             }
             default -> throw new UnsupportedOperationException();
         }
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomValue));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", randomValue)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -536,7 +536,7 @@ public void testAutoConvertNotString() throws Exception {
 
     public void testAutoConvertStringNotMatched() throws Exception {
         String value = "notAnIntFloatOrBool";
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", value));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", value)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -546,7 +546,7 @@ public void testAutoConvertStringNotMatched() throws Exception {
     public void testAutoConvertMatchBoolean() throws Exception {
         boolean randomBoolean = randomBoolean();
         String booleanString = Boolean.toString(randomBoolean);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", booleanString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", booleanString)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -556,7 +556,7 @@ public void testAutoConvertMatchBoolean() throws Exception {
     public void testAutoConvertMatchInteger() throws Exception {
         int randomInt = randomInt();
         String randomString = Integer.toString(randomInt);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", randomString)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -566,7 +566,7 @@ public void testAutoConvertMatchInteger() throws Exception {
     public void testAutoConvertMatchLong() throws Exception {
         long randomLong = randomLong();
         String randomString = Long.toString(randomLong);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", randomString)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -577,7 +577,7 @@ public void testAutoConvertDoubleNotMatched() throws Exception {
         double randomDouble = randomDouble();
         String randomString = Double.toString(randomDouble);
         float randomFloat = Float.parseFloat(randomString);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", randomString)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -588,7 +588,7 @@ public void testAutoConvertDoubleNotMatched() throws Exception {
     public void testAutoConvertMatchFloat() throws Exception {
         float randomFloat = randomFloat();
         String randomString = Float.toString(randomFloat);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("field", randomString)));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);